2024-11-14 00:27:15,662 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 00:27:15,688 main DEBUG Took 0.024063 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 00:27:15,689 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 00:27:15,690 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 00:27:15,691 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 00:27:15,693 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 00:27:15,705 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 00:27:15,734 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 00:27:15,736 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 00:27:15,737 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 00:27:15,737 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 00:27:15,738 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 00:27:15,739 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 00:27:15,740 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 00:27:15,741 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 00:27:15,742 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 00:27:15,751 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 00:27:15,752 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 00:27:15,753 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 00:27:15,754 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 00:27:15,754 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 00:27:15,756 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 00:27:15,756 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 00:27:15,758 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 00:27:15,758 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 00:27:15,762 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 00:27:15,763 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 00:27:15,764 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 00:27:15,764 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 00:27:15,765 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 00:27:15,766 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 00:27:15,766 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 00:27:15,767 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 00:27:15,770 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 00:27:15,772 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 00:27:15,777 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 00:27:15,778 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 00:27:15,780 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 00:27:15,781 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 00:27:15,792 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 00:27:15,794 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 00:27:15,797 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 00:27:15,797 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 00:27:15,797 main DEBUG createAppenders(={Console}) 2024-11-14 00:27:15,798 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-14 00:27:15,798 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-14 00:27:15,799 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-14 00:27:15,800 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 00:27:15,800 main DEBUG OutputStream closed 2024-11-14 00:27:15,801 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 00:27:15,801 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 00:27:15,801 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-14 00:27:15,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 00:27:15,910 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 00:27:15,911 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 00:27:15,913 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 00:27:15,913 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 00:27:15,913 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 00:27:15,914 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 00:27:15,914 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 00:27:15,914 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 00:27:15,915 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 00:27:15,915 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 00:27:15,916 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 00:27:15,916 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 00:27:15,916 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 00:27:15,917 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 00:27:15,917 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 00:27:15,917 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 00:27:15,918 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 00:27:15,920 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 00:27:15,921 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-14 00:27:15,922 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 00:27:15,923 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-14T00:27:16,267 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5 2024-11-14 00:27:16,271 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 00:27:16,271 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T00:27:16,283 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-14T00:27:16,311 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T00:27:16,335 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/cluster_41f6304e-0c68-787b-c426-53235df1090e, deleteOnExit=true 2024-11-14T00:27:16,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T00:27:16,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/test.cache.data in system properties and HBase conf 2024-11-14T00:27:16,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T00:27:16,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/hadoop.log.dir in system properties and HBase conf 2024-11-14T00:27:16,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T00:27:16,343 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T00:27:16,344 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T00:27:16,510 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T00:27:16,681 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T00:27:16,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T00:27:16,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T00:27:16,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T00:27:16,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T00:27:16,695 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T00:27:16,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T00:27:16,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T00:27:16,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T00:27:16,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T00:27:16,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/nfs.dump.dir in system properties and HBase conf 2024-11-14T00:27:16,711 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/java.io.tmpdir in system properties and HBase conf 2024-11-14T00:27:16,711 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T00:27:16,712 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T00:27:16,712 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T00:27:18,646 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T00:27:18,726 INFO [Time-limited test {}] log.Log(170): Logging initialized @4486ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T00:27:18,817 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T00:27:18,916 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T00:27:18,969 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T00:27:18,972 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T00:27:18,974 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T00:27:18,993 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T00:27:18,997 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@610d6b27{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/hadoop.log.dir/,AVAILABLE} 2024-11-14T00:27:18,999 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31a6a643{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T00:27:19,337 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7f405bdd{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/java.io.tmpdir/jetty-localhost-37751-hadoop-hdfs-3_4_1-tests_jar-_-any-12022601350782426682/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T00:27:19,351 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5b829fcb{HTTP/1.1, (http/1.1)}{localhost:37751} 2024-11-14T00:27:19,351 INFO [Time-limited test {}] server.Server(415): Started @5112ms 2024-11-14T00:27:20,479 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T00:27:20,499 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T00:27:20,516 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T00:27:20,524 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T00:27:20,525 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T00:27:20,544 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@24457f8e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/hadoop.log.dir/,AVAILABLE} 2024-11-14T00:27:20,545 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b5418b0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T00:27:20,743 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@46a94ab{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/java.io.tmpdir/jetty-localhost-40613-hadoop-hdfs-3_4_1-tests_jar-_-any-5277942589168208570/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T00:27:20,745 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@753587b3{HTTP/1.1, (http/1.1)}{localhost:40613} 2024-11-14T00:27:20,746 INFO [Time-limited test {}] server.Server(415): Started @6507ms 2024-11-14T00:27:20,850 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T00:27:22,999 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/cluster_41f6304e-0c68-787b-c426-53235df1090e/data/data2/current/BP-301748371-172.17.0.3-1731544037581/current, will proceed with Du for space computation calculation, 2024-11-14T00:27:23,000 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/cluster_41f6304e-0c68-787b-c426-53235df1090e/data/data1/current/BP-301748371-172.17.0.3-1731544037581/current, will proceed with Du for space computation calculation, 2024-11-14T00:27:23,071 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T00:27:23,142 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x26cd63271ff4ddb with lease ID 0xcbef973635bede32: Processing first storage report for DS-af742712-73c0-4671-b2ca-28562fffbdf4 from datanode DatanodeRegistration(127.0.0.1:43405, datanodeUuid=16a71a9d-0b02-43b0-8a7a-4cb8cb1ab9d4, infoPort=38711, infoSecurePort=0, ipcPort=46165, storageInfo=lv=-57;cid=testClusterID;nsid=2092749499;c=1731544037581) 2024-11-14T00:27:23,144 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x26cd63271ff4ddb with lease ID 0xcbef973635bede32: from storage DS-af742712-73c0-4671-b2ca-28562fffbdf4 node DatanodeRegistration(127.0.0.1:43405, datanodeUuid=16a71a9d-0b02-43b0-8a7a-4cb8cb1ab9d4, infoPort=38711, infoSecurePort=0, ipcPort=46165, storageInfo=lv=-57;cid=testClusterID;nsid=2092749499;c=1731544037581), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-14T00:27:23,147 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x26cd63271ff4ddb with lease ID 0xcbef973635bede32: Processing first storage report for DS-0582548a-08de-4bb8-b95a-a0ab26999d84 from datanode DatanodeRegistration(127.0.0.1:43405, datanodeUuid=16a71a9d-0b02-43b0-8a7a-4cb8cb1ab9d4, infoPort=38711, infoSecurePort=0, ipcPort=46165, storageInfo=lv=-57;cid=testClusterID;nsid=2092749499;c=1731544037581) 2024-11-14T00:27:23,147 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x26cd63271ff4ddb with lease ID 0xcbef973635bede32: from storage DS-0582548a-08de-4bb8-b95a-a0ab26999d84 node DatanodeRegistration(127.0.0.1:43405, datanodeUuid=16a71a9d-0b02-43b0-8a7a-4cb8cb1ab9d4, infoPort=38711, infoSecurePort=0, ipcPort=46165, storageInfo=lv=-57;cid=testClusterID;nsid=2092749499;c=1731544037581), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T00:27:23,210 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5 2024-11-14T00:27:23,335 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/cluster_41f6304e-0c68-787b-c426-53235df1090e/zookeeper_0, clientPort=56442, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/cluster_41f6304e-0c68-787b-c426-53235df1090e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/cluster_41f6304e-0c68-787b-c426-53235df1090e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T00:27:23,357 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56442 2024-11-14T00:27:23,374 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T00:27:23,379 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T00:27:23,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741825_1001 (size=7) 2024-11-14T00:27:24,148 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c with version=8 2024-11-14T00:27:24,148 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/hbase-staging 2024-11-14T00:27:24,274 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T00:27:24,587 INFO [Time-limited test {}] client.ConnectionUtils(128): master/63baa11b75f3:0 server-side Connection retries=45 2024-11-14T00:27:24,601 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T00:27:24,602 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T00:27:24,611 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T00:27:24,612 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T00:27:24,614 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T00:27:24,827 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T00:27:24,912 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T00:27:24,924 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T00:27:24,928 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T00:27:24,965 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 34267 (auto-detected) 2024-11-14T00:27:24,966 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-14T00:27:24,992 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36271 2024-11-14T00:27:25,021 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36271 connecting to ZooKeeper ensemble=127.0.0.1:56442 2024-11-14T00:27:25,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:362710x0, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T00:27:25,126 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36271-0x10136bd4ad10000 connected 2024-11-14T00:27:25,226 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T00:27:25,228 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T00:27:25,239 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T00:27:25,243 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c, hbase.cluster.distributed=false 2024-11-14T00:27:25,268 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T00:27:25,276 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36271 2024-11-14T00:27:25,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36271 2024-11-14T00:27:25,282 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36271 2024-11-14T00:27:25,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36271 2024-11-14T00:27:25,293 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36271 2024-11-14T00:27:25,431 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/63baa11b75f3:0 server-side Connection retries=45 2024-11-14T00:27:25,433 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T00:27:25,434 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T00:27:25,434 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T00:27:25,434 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T00:27:25,434 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T00:27:25,437 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T00:27:25,440 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T00:27:25,441 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33199 2024-11-14T00:27:25,443 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33199 connecting to ZooKeeper ensemble=127.0.0.1:56442 2024-11-14T00:27:25,445 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T00:27:25,447 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T00:27:25,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:331990x0, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T00:27:25,465 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:331990x0, quorum=127.0.0.1:56442, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T00:27:25,466 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33199-0x10136bd4ad10001 connected 2024-11-14T00:27:25,471 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T00:27:25,487 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T00:27:25,497 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T00:27:25,504 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T00:27:25,512 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33199 2024-11-14T00:27:25,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33199 2024-11-14T00:27:25,525 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33199 2024-11-14T00:27:25,540 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33199 2024-11-14T00:27:25,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33199 2024-11-14T00:27:25,575 DEBUG [M:0;63baa11b75f3:36271 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;63baa11b75f3:36271 2024-11-14T00:27:25,581 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/63baa11b75f3,36271,1731544044347 2024-11-14T00:27:25,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T00:27:25,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T00:27:25,610 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/63baa11b75f3,36271,1731544044347 2024-11-14T00:27:25,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T00:27:25,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T00:27:25,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T00:27:25,676 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T00:27:25,688 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/63baa11b75f3,36271,1731544044347 from backup master directory 2024-11-14T00:27:25,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T00:27:25,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/63baa11b75f3,36271,1731544044347 2024-11-14T00:27:25,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T00:27:25,710 WARN [master/63baa11b75f3:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T00:27:25,711 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=63baa11b75f3,36271,1731544044347 2024-11-14T00:27:25,726 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T00:27:25,730 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T00:27:25,837 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/hbase.id] with ID: 65c5933d-f8ea-4873-8a51-cc20325705b8 2024-11-14T00:27:25,837 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/.tmp/hbase.id 2024-11-14T00:27:25,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741826_1002 (size=42) 2024-11-14T00:27:25,857 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/.tmp/hbase.id]:[hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/hbase.id] 2024-11-14T00:27:25,947 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T00:27:25,957 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T00:27:25,983 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-14T00:27:26,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T00:27:26,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T00:27:26,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741827_1003 (size=196) 2024-11-14T00:27:26,073 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T00:27:26,081 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T00:27:26,108 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:26,114 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T00:27:26,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741828_1004 (size=1189) 2024-11-14T00:27:26,644 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store 2024-11-14T00:27:26,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741829_1005 (size=34) 2024-11-14T00:27:26,705 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T00:27:26,708 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:27:26,710 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T00:27:26,710 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T00:27:26,710 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T00:27:26,712 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T00:27:26,712 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T00:27:26,712 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T00:27:26,714 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731544046710Disabling compacts and flushes for region at 1731544046710Disabling writes for close at 1731544046712 (+2 ms)Writing region close event to WAL at 1731544046712Closed at 1731544046712 2024-11-14T00:27:26,716 WARN [master/63baa11b75f3:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/.initializing 2024-11-14T00:27:26,717 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/WALs/63baa11b75f3,36271,1731544044347 2024-11-14T00:27:26,727 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T00:27:26,762 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=63baa11b75f3%2C36271%2C1731544044347, suffix=, logDir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/WALs/63baa11b75f3,36271,1731544044347, archiveDir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/oldWALs, maxLogs=10 2024-11-14T00:27:26,811 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/WALs/63baa11b75f3,36271,1731544044347/63baa11b75f3%2C36271%2C1731544044347.1731544046770, exclude list is [], retry=0 2024-11-14T00:27:26,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43405,DS-af742712-73c0-4671-b2ca-28562fffbdf4,DISK] 2024-11-14T00:27:26,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T00:27:26,908 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/WALs/63baa11b75f3,36271,1731544044347/63baa11b75f3%2C36271%2C1731544044347.1731544046770 2024-11-14T00:27:26,910 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38711:38711)] 2024-11-14T00:27:26,911 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T00:27:26,911 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:27:26,916 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T00:27:26,918 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T00:27:26,966 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T00:27:27,015 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T00:27:27,022 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:27,028 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T00:27:27,029 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T00:27:27,036 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T00:27:27,037 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:27,042 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:27:27,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T00:27:27,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T00:27:27,054 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:27,059 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:27:27,060 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T00:27:27,065 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T00:27:27,065 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:27,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:27:27,067 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T00:27:27,077 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T00:27:27,078 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T00:27:27,086 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T00:27:27,087 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T00:27:27,104 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T00:27:27,115 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T00:27:27,131 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T00:27:27,144 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63944666, jitterRate=-0.047150224447250366}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T00:27:27,157 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731544046934Initializing all the Stores at 1731544046937 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731544046938 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544046940 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544046940Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544046941 (+1 ms)Cleaning up temporary data from old regions at 1731544047087 (+146 ms)Region opened successfully at 1731544047157 (+70 ms) 2024-11-14T00:27:27,159 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T00:27:27,219 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@587858ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=63baa11b75f3/172.17.0.3:0 2024-11-14T00:27:27,264 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T00:27:27,280 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T00:27:27,280 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T00:27:27,287 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T00:27:27,290 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 3 msec 2024-11-14T00:27:27,299 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-14T00:27:27,299 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T00:27:27,360 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T00:27:27,385 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T00:27:27,466 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T00:27:27,469 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T00:27:27,476 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T00:27:27,496 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T00:27:27,500 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T00:27:27,511 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T00:27:27,526 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T00:27:27,529 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T00:27:27,539 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T00:27:27,567 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T00:27:27,580 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T00:27:27,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T00:27:27,596 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=63baa11b75f3,36271,1731544044347, sessionid=0x10136bd4ad10000, setting cluster-up flag (Was=false) 2024-11-14T00:27:27,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T00:27:27,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T00:27:27,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T00:27:27,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T00:27:27,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T00:27:27,710 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T00:27:27,733 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=63baa11b75f3,36271,1731544044347 2024-11-14T00:27:27,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T00:27:27,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T00:27:27,803 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T00:27:27,810 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=63baa11b75f3,36271,1731544044347 2024-11-14T00:27:27,824 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T00:27:27,905 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(746): ClusterId : 65c5933d-f8ea-4873-8a51-cc20325705b8 2024-11-14T00:27:27,908 DEBUG [RS:0;63baa11b75f3:33199 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T00:27:27,975 DEBUG [RS:0;63baa11b75f3:33199 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T00:27:27,975 DEBUG [RS:0;63baa11b75f3:33199 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T00:27:28,008 DEBUG [RS:0;63baa11b75f3:33199 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T00:27:28,012 DEBUG [RS:0;63baa11b75f3:33199 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@158997ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=63baa11b75f3/172.17.0.3:0 2024-11-14T00:27:28,046 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T00:27:28,067 DEBUG [RS:0;63baa11b75f3:33199 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;63baa11b75f3:33199 2024-11-14T00:27:28,079 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T00:27:28,086 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T00:27:28,086 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T00:27:28,088 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T00:27:28,086 DEBUG [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T00:27:28,098 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 63baa11b75f3,36271,1731544044347 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T00:27:28,107 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(2659): reportForDuty to master=63baa11b75f3,36271,1731544044347 with port=33199, startcode=1731544045387 2024-11-14T00:27:28,121 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/63baa11b75f3:0, corePoolSize=5, maxPoolSize=5 2024-11-14T00:27:28,122 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/63baa11b75f3:0, corePoolSize=5, maxPoolSize=5 2024-11-14T00:27:28,122 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/63baa11b75f3:0, corePoolSize=5, maxPoolSize=5 2024-11-14T00:27:28,122 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/63baa11b75f3:0, corePoolSize=5, maxPoolSize=5 2024-11-14T00:27:28,122 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/63baa11b75f3:0, corePoolSize=10, maxPoolSize=10 2024-11-14T00:27:28,122 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/63baa11b75f3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T00:27:28,123 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/63baa11b75f3:0, corePoolSize=2, maxPoolSize=2 2024-11-14T00:27:28,123 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/63baa11b75f3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T00:27:28,126 DEBUG [RS:0;63baa11b75f3:33199 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T00:27:28,143 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731544078142 2024-11-14T00:27:28,145 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T00:27:28,146 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T00:27:28,146 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T00:27:28,148 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T00:27:28,150 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T00:27:28,150 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T00:27:28,151 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T00:27:28,151 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T00:27:28,164 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:28,173 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:28,174 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T00:27:28,177 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T00:27:28,179 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T00:27:28,179 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T00:27:28,191 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T00:27:28,192 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T00:27:28,196 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34155, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T00:27:28,200 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/63baa11b75f3:0:becomeActiveMaster-HFileCleaner.large.0-1731544048193,5,FailOnTimeoutGroup] 2024-11-14T00:27:28,204 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/63baa11b75f3:0:becomeActiveMaster-HFileCleaner.small.0-1731544048200,5,FailOnTimeoutGroup] 2024-11-14T00:27:28,204 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:28,205 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T00:27:28,206 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:28,207 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:28,208 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36271 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 63baa11b75f3,33199,1731544045387 2024-11-14T00:27:28,211 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36271 {}] master.ServerManager(517): Registering regionserver=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:28,233 DEBUG [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c 2024-11-14T00:27:28,233 DEBUG [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46041 2024-11-14T00:27:28,234 DEBUG [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T00:27:28,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741831_1007 (size=1321) 2024-11-14T00:27:28,241 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T00:27:28,242 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c 2024-11-14T00:27:28,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741832_1008 (size=32) 2024-11-14T00:27:28,282 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:27:28,285 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T00:27:28,290 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T00:27:28,290 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:28,292 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T00:27:28,292 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T00:27:28,297 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T00:27:28,297 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:28,299 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T00:27:28,300 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T00:27:28,305 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T00:27:28,305 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:28,307 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T00:27:28,307 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T00:27:28,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T00:27:28,321 DEBUG [RS:0;63baa11b75f3:33199 {}] zookeeper.ZKUtil(111): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/63baa11b75f3,33199,1731544045387 2024-11-14T00:27:28,321 WARN [RS:0;63baa11b75f3:33199 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T00:27:28,322 INFO [RS:0;63baa11b75f3:33199 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T00:27:28,322 DEBUG [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/WALs/63baa11b75f3,33199,1731544045387 2024-11-14T00:27:28,336 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T00:27:28,337 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:28,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T00:27:28,342 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T00:27:28,345 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740 2024-11-14T00:27:28,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740 2024-11-14T00:27:28,352 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [63baa11b75f3,33199,1731544045387] 2024-11-14T00:27:28,377 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T00:27:28,389 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T00:27:28,414 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T00:27:28,428 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T00:27:28,447 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T00:27:28,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T00:27:28,469 INFO [RS:0;63baa11b75f3:33199 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T00:27:28,470 INFO [RS:0;63baa11b75f3:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:28,474 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T00:27:28,489 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T00:27:28,491 INFO [RS:0;63baa11b75f3:33199 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T00:27:28,491 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65559725, jitterRate=-0.023083969950675964}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T00:27:28,493 INFO [RS:0;63baa11b75f3:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:28,495 DEBUG [RS:0;63baa11b75f3:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/63baa11b75f3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T00:27:28,495 DEBUG [RS:0;63baa11b75f3:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/63baa11b75f3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T00:27:28,495 DEBUG [RS:0;63baa11b75f3:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/63baa11b75f3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T00:27:28,495 DEBUG [RS:0;63baa11b75f3:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/63baa11b75f3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T00:27:28,495 DEBUG [RS:0;63baa11b75f3:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/63baa11b75f3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T00:27:28,496 DEBUG [RS:0;63baa11b75f3:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/63baa11b75f3:0, corePoolSize=2, maxPoolSize=2 2024-11-14T00:27:28,496 DEBUG [RS:0;63baa11b75f3:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/63baa11b75f3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T00:27:28,496 DEBUG [RS:0;63baa11b75f3:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/63baa11b75f3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T00:27:28,498 DEBUG [RS:0;63baa11b75f3:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/63baa11b75f3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T00:27:28,498 DEBUG [RS:0;63baa11b75f3:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/63baa11b75f3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T00:27:28,498 DEBUG [RS:0;63baa11b75f3:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/63baa11b75f3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T00:27:28,498 DEBUG [RS:0;63baa11b75f3:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/63baa11b75f3:0, corePoolSize=1, maxPoolSize=1 2024-11-14T00:27:28,498 DEBUG [RS:0;63baa11b75f3:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/63baa11b75f3:0, corePoolSize=3, maxPoolSize=3 2024-11-14T00:27:28,499 DEBUG [RS:0;63baa11b75f3:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0, corePoolSize=3, maxPoolSize=3 2024-11-14T00:27:28,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731544048282Initializing all the Stores at 1731544048285 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731544048285Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731544048285Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544048285Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731544048285Cleaning up temporary data from old regions at 1731544048420 (+135 ms)Region opened successfully at 1731544048498 (+78 ms) 2024-11-14T00:27:28,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T00:27:28,499 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T00:27:28,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T00:27:28,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T00:27:28,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T00:27:28,505 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T00:27:28,506 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731544048499Disabling compacts and flushes for region at 1731544048499Disabling writes for close at 1731544048499Writing region close event to WAL at 1731544048505 (+6 ms)Closed at 1731544048505 2024-11-14T00:27:28,513 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T00:27:28,513 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T00:27:28,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T00:27:28,532 INFO [RS:0;63baa11b75f3:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:28,532 INFO [RS:0;63baa11b75f3:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:28,533 INFO [RS:0;63baa11b75f3:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:28,533 INFO [RS:0;63baa11b75f3:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:28,533 INFO [RS:0;63baa11b75f3:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:28,533 INFO [RS:0;63baa11b75f3:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=63baa11b75f3,33199,1731544045387-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T00:27:28,540 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T00:27:28,549 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T00:27:28,566 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T00:27:28,571 INFO [RS:0;63baa11b75f3:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=63baa11b75f3,33199,1731544045387-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:28,572 INFO [RS:0;63baa11b75f3:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:28,576 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.Replication(171): 63baa11b75f3,33199,1731544045387 started 2024-11-14T00:27:28,653 INFO [RS:0;63baa11b75f3:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:28,654 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1482): Serving as 63baa11b75f3,33199,1731544045387, RpcServer on 63baa11b75f3/172.17.0.3:33199, sessionid=0x10136bd4ad10001 2024-11-14T00:27:28,655 DEBUG [RS:0;63baa11b75f3:33199 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T00:27:28,655 DEBUG [RS:0;63baa11b75f3:33199 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 63baa11b75f3,33199,1731544045387 2024-11-14T00:27:28,655 DEBUG [RS:0;63baa11b75f3:33199 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '63baa11b75f3,33199,1731544045387' 2024-11-14T00:27:28,656 DEBUG [RS:0;63baa11b75f3:33199 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T00:27:28,657 DEBUG [RS:0;63baa11b75f3:33199 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T00:27:28,659 DEBUG [RS:0;63baa11b75f3:33199 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T00:27:28,659 DEBUG [RS:0;63baa11b75f3:33199 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T00:27:28,660 DEBUG [RS:0;63baa11b75f3:33199 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 63baa11b75f3,33199,1731544045387 2024-11-14T00:27:28,660 DEBUG [RS:0;63baa11b75f3:33199 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '63baa11b75f3,33199,1731544045387' 2024-11-14T00:27:28,660 DEBUG [RS:0;63baa11b75f3:33199 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T00:27:28,666 DEBUG [RS:0;63baa11b75f3:33199 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T00:27:28,672 DEBUG [RS:0;63baa11b75f3:33199 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T00:27:28,672 INFO [RS:0;63baa11b75f3:33199 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T00:27:28,673 INFO [RS:0;63baa11b75f3:33199 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T00:27:28,701 WARN [63baa11b75f3:36271 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-14T00:27:28,779 INFO [RS:0;63baa11b75f3:33199 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T00:27:28,793 INFO [RS:0;63baa11b75f3:33199 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=63baa11b75f3%2C33199%2C1731544045387, suffix=, logDir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/WALs/63baa11b75f3,33199,1731544045387, archiveDir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/oldWALs, maxLogs=32 2024-11-14T00:27:28,831 DEBUG [RS:0;63baa11b75f3:33199 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/WALs/63baa11b75f3,33199,1731544045387/63baa11b75f3%2C33199%2C1731544045387.1731544048797, exclude list is [], retry=0 2024-11-14T00:27:28,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43405,DS-af742712-73c0-4671-b2ca-28562fffbdf4,DISK] 2024-11-14T00:27:28,897 INFO [RS:0;63baa11b75f3:33199 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/WALs/63baa11b75f3,33199,1731544045387/63baa11b75f3%2C33199%2C1731544045387.1731544048797 2024-11-14T00:27:28,908 DEBUG [RS:0;63baa11b75f3:33199 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38711:38711)] 2024-11-14T00:27:28,954 DEBUG [63baa11b75f3:36271 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T00:27:28,975 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:28,984 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 63baa11b75f3,33199,1731544045387, state=OPENING 2024-11-14T00:27:29,013 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T00:27:29,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T00:27:29,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T00:27:29,047 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T00:27:29,055 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T00:27:29,056 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T00:27:29,058 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:27:29,249 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T00:27:29,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51983, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T00:27:29,276 INFO [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T00:27:29,276 INFO [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T00:27:29,277 INFO [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T00:27:29,291 INFO [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=63baa11b75f3%2C33199%2C1731544045387.meta, suffix=.meta, logDir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/WALs/63baa11b75f3,33199,1731544045387, archiveDir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/oldWALs, maxLogs=32 2024-11-14T00:27:29,314 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/WALs/63baa11b75f3,33199,1731544045387/63baa11b75f3%2C33199%2C1731544045387.meta.1731544049294.meta, exclude list is [], retry=0 2024-11-14T00:27:29,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43405,DS-af742712-73c0-4671-b2ca-28562fffbdf4,DISK] 2024-11-14T00:27:29,341 INFO [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/WALs/63baa11b75f3,33199,1731544045387/63baa11b75f3%2C33199%2C1731544045387.meta.1731544049294.meta 2024-11-14T00:27:29,348 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38711:38711)] 2024-11-14T00:27:29,349 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T00:27:29,350 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T00:27:29,354 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T00:27:29,361 INFO [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T00:27:29,367 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T00:27:29,370 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:27:29,370 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T00:27:29,371 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T00:27:29,399 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T00:27:29,402 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T00:27:29,402 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:29,403 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T00:27:29,403 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T00:27:29,406 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T00:27:29,406 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:29,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T00:27:29,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T00:27:29,411 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T00:27:29,411 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:29,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T00:27:29,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T00:27:29,415 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T00:27:29,415 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:29,416 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T00:27:29,416 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T00:27:29,424 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740 2024-11-14T00:27:29,430 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740 2024-11-14T00:27:29,433 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T00:27:29,433 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T00:27:29,434 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T00:27:29,437 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T00:27:29,439 INFO [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60553014, jitterRate=-0.09768977761268616}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T00:27:29,439 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T00:27:29,441 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731544049372Writing region info on filesystem at 1731544049372Initializing all the Stores at 1731544049381 (+9 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731544049381Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731544049397 (+16 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544049397Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731544049397Cleaning up temporary data from old regions at 1731544049433 (+36 ms)Running coprocessor post-open hooks at 1731544049439 (+6 ms)Region opened successfully at 1731544049441 (+2 ms) 2024-11-14T00:27:29,449 INFO [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731544049238 2024-11-14T00:27:29,461 DEBUG [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T00:27:29,461 INFO [RS_OPEN_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T00:27:29,464 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:29,469 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 63baa11b75f3,33199,1731544045387, state=OPEN 2024-11-14T00:27:29,517 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:29,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T00:27:29,517 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T00:27:29,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T00:27:29,521 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T00:27:29,526 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T00:27:29,526 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=63baa11b75f3,33199,1731544045387 in 459 msec 2024-11-14T00:27:29,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T00:27:29,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0080 sec 2024-11-14T00:27:29,546 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T00:27:29,546 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T00:27:29,578 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:29,579 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:29,610 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:29,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47367, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:29,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7710 sec 2024-11-14T00:27:29,668 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731544049668, completionTime=-1 2024-11-14T00:27:29,671 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T00:27:29,671 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T00:27:29,808 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-14T00:27:29,808 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731544109808 2024-11-14T00:27:29,808 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731544169808 2024-11-14T00:27:29,808 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 137 msec 2024-11-14T00:27:29,826 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=63baa11b75f3,36271,1731544044347-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:29,826 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=63baa11b75f3,36271,1731544044347-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:29,826 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=63baa11b75f3,36271,1731544044347-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:29,828 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-63baa11b75f3:36271, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:29,829 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:29,835 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:29,839 DEBUG [master/63baa11b75f3:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T00:27:29,921 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.210sec 2024-11-14T00:27:29,923 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T00:27:29,936 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T00:27:29,938 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T00:27:29,939 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T00:27:29,939 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T00:27:29,942 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=63baa11b75f3,36271,1731544044347-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T00:27:29,943 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=63baa11b75f3,36271,1731544044347-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T00:27:29,961 DEBUG [master/63baa11b75f3:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T00:27:29,962 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T00:27:29,964 INFO [master/63baa11b75f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=63baa11b75f3,36271,1731544044347-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T00:27:29,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@534da798, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:29,992 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T00:27:29,993 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T00:27:29,997 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:30,000 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:30,013 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:30,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:30,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424d861b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:30,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:30,051 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:30,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2be54e96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:30,061 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:30,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:30,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:30,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=63baa11b75f3,36271,1731544044347 2024-11-14T00:27:30,149 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=1140, ProcessCount=11, AvailableMemoryMB=3687 2024-11-14T00:27:30,166 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T00:27:30,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 63baa11b75f3,36271,1731544044347 2024-11-14T00:27:30,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b888961 2024-11-14T00:27:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T00:27:30,189 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T00:27:30,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T00:27:30,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T00:27:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T00:27:30,214 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T00:27:30,217 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-14T00:27:30,217 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:30,222 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T00:27:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T00:27:30,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741835_1011 (size=959) 2024-11-14T00:27:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T00:27:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T00:27:30,670 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c 2024-11-14T00:27:30,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741836_1012 (size=53) 2024-11-14T00:27:30,687 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:27:30,687 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 285207bf1146874945959d9159bd86b2, disabling compactions & flushes 2024-11-14T00:27:30,687 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:30,687 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:30,687 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. after waiting 0 ms 2024-11-14T00:27:30,688 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:30,688 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:30,688 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 285207bf1146874945959d9159bd86b2: Waiting for close lock at 1731544050687Disabling compacts and flushes for region at 1731544050687Disabling writes for close at 1731544050688 (+1 ms)Writing region close event to WAL at 1731544050688Closed at 1731544050688 2024-11-14T00:27:30,694 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T00:27:30,700 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731544050694"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731544050694"}]},"ts":"1731544050694"} 2024-11-14T00:27:30,706 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T00:27:30,709 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T00:27:30,712 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544050709"}]},"ts":"1731544050709"} 2024-11-14T00:27:30,718 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T00:27:30,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=285207bf1146874945959d9159bd86b2, ASSIGN}] 2024-11-14T00:27:30,723 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=285207bf1146874945959d9159bd86b2, ASSIGN 2024-11-14T00:27:30,726 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=285207bf1146874945959d9159bd86b2, ASSIGN; state=OFFLINE, location=63baa11b75f3,33199,1731544045387; forceNewPlan=false, retain=false 2024-11-14T00:27:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T00:27:30,883 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=285207bf1146874945959d9159bd86b2, regionState=OPENING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:30,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=285207bf1146874945959d9159bd86b2, ASSIGN because future has completed 2024-11-14T00:27:30,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:27:31,088 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:31,089 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} 2024-11-14T00:27:31,089 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:31,090 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:27:31,090 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:31,090 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:31,108 INFO [StoreOpener-285207bf1146874945959d9159bd86b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:31,117 INFO [StoreOpener-285207bf1146874945959d9159bd86b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 285207bf1146874945959d9159bd86b2 columnFamilyName A 2024-11-14T00:27:31,117 DEBUG [StoreOpener-285207bf1146874945959d9159bd86b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:31,121 INFO [StoreOpener-285207bf1146874945959d9159bd86b2-1 {}] regionserver.HStore(327): Store=285207bf1146874945959d9159bd86b2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:27:31,121 INFO [StoreOpener-285207bf1146874945959d9159bd86b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:31,130 INFO [StoreOpener-285207bf1146874945959d9159bd86b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 285207bf1146874945959d9159bd86b2 columnFamilyName B 2024-11-14T00:27:31,130 DEBUG [StoreOpener-285207bf1146874945959d9159bd86b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:31,131 INFO [StoreOpener-285207bf1146874945959d9159bd86b2-1 {}] regionserver.HStore(327): Store=285207bf1146874945959d9159bd86b2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:27:31,132 INFO [StoreOpener-285207bf1146874945959d9159bd86b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:31,136 INFO [StoreOpener-285207bf1146874945959d9159bd86b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 285207bf1146874945959d9159bd86b2 columnFamilyName C 2024-11-14T00:27:31,136 DEBUG [StoreOpener-285207bf1146874945959d9159bd86b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:31,138 INFO [StoreOpener-285207bf1146874945959d9159bd86b2-1 {}] regionserver.HStore(327): Store=285207bf1146874945959d9159bd86b2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:27:31,138 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:31,140 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2 2024-11-14T00:27:31,142 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2 2024-11-14T00:27:31,150 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:31,150 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:31,157 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T00:27:31,162 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:31,192 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T00:27:31,197 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 285207bf1146874945959d9159bd86b2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64826192, jitterRate=-0.03401446342468262}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T00:27:31,197 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:31,199 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 285207bf1146874945959d9159bd86b2: Running coprocessor pre-open hook at 1731544051090Writing region info on filesystem at 1731544051090Initializing all the Stores at 1731544051104 (+14 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544051104Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544051108 (+4 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544051108Cleaning up temporary data from old regions at 1731544051150 (+42 ms)Running coprocessor post-open hooks at 1731544051197 (+47 ms)Region opened successfully at 1731544051199 (+2 ms) 2024-11-14T00:27:31,221 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., pid=6, masterSystemTime=1731544051059 2024-11-14T00:27:31,230 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:31,230 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:31,253 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=285207bf1146874945959d9159bd86b2, regionState=OPEN, openSeqNum=2, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:31,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:27:31,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T00:27:31,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=285207bf1146874945959d9159bd86b2, ASSIGN in 577 msec 2024-11-14T00:27:31,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T00:27:31,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 in 383 msec 2024-11-14T00:27:31,314 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T00:27:31,314 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544051314"}]},"ts":"1731544051314"} 2024-11-14T00:27:31,333 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T00:27:31,336 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T00:27:31,343 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1340 sec 2024-11-14T00:27:31,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T00:27:31,386 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:27:31,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24cd370b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:31,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:31,429 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:31,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:31,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:31,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dae95e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:31,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:31,449 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:31,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50fdeca0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,454 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:31,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:31,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:31,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:31,476 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T00:27:31,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 63baa11b75f3,36271,1731544044347 2024-11-14T00:27:31,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3ea317c2 2024-11-14T00:27:31,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T00:27:31,505 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T00:27:31,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a25f29a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,518 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:31,519 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:31,530 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:31,530 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:31,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:31,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@455d50bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:31,532 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:31,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:31,553 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:31,561 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@760c0989, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8953fc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,565 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:31,565 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:31,574 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:31,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:31,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:31,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3d2fd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:31,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:31,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:31,599 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c860641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75df9ef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,617 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:31,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:31,630 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:31,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cdb8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:31,639 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:31,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b21050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5731bbe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:31,646 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:31,678 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:31,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b084ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:31,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:31,693 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2815dd8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c885828, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:31,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:31,705 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@746f284d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:31,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:31,712 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:31,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63944c38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28287983, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:31,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:31,718 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e9d3a67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:31,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:31,725 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b5a398e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c04eae4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:31,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:31,742 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56fd7393, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:31,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:31,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:31,750 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:31,752 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69fb1ae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d75e95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:31,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:31,776 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26e2c10d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:31,778 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:31,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@739b8c1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c9d7e50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:31,781 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:31,782 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:31,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:31,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@355546b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:31,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:31,783 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:31,787 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23600c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:31,799 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:31,801 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:31,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:31,806 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:31,806 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:31,807 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:31,808 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:31,808 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:31,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:31,809 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:27:31,809 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:31,812 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:31,812 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:31,813 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:31,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:31,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:31,815 DEBUG [Thread-147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:31,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:31,818 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:31,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:31,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:31,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:31,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:31,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:31,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:31,824 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:31,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:27:31,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:27:31,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:27:31,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:27:31,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-14T00:27:31,831 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:27:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T00:27:31,840 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:27:31,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:27:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:27:31,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:27:31,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:27:31,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:31,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:27:31,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:27:31,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T00:27:31,964 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:27:31,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T00:27:31,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:32,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T00:27:32,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:32,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:32,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:32,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/790dc60e3a3d43aead8f442ee25d4abd is 50, key is test_row_0/A:col10/1731544051960/Put/seqid=0 2024-11-14T00:27:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T00:27:32,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544112208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T00:27:32,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544112215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544112190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544112222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:32,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:32,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:32,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741837_1013 (size=12001) 2024-11-14T00:27:32,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/790dc60e3a3d43aead8f442ee25d4abd 2024-11-14T00:27:32,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544112234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544112389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544112391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544112391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544112393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544112392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T00:27:32,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:32,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:32,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:32,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T00:27:32,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/9e2a998618e24966abcb1665ee095191 is 50, key is test_row_0/B:col10/1731544051960/Put/seqid=0 2024-11-14T00:27:32,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741838_1014 (size=12001) 2024-11-14T00:27:32,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/9e2a998618e24966abcb1665ee095191 2024-11-14T00:27:32,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T00:27:32,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:32,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:32,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:32,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/1d71d3cca089429697d99cffd0e5115a is 50, key is test_row_0/C:col10/1731544051960/Put/seqid=0 2024-11-14T00:27:32,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544112597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544112617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544112635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544112635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544112636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741839_1015 (size=12001) 2024-11-14T00:27:32,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T00:27:32,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:32,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544112916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T00:27:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:32,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:32,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544112945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544112955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544112960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544112961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T00:27:33,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/1d71d3cca089429697d99cffd0e5115a 2024-11-14T00:27:33,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T00:27:33,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:33,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:33,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:33,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:33,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:33,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/790dc60e3a3d43aead8f442ee25d4abd as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/790dc60e3a3d43aead8f442ee25d4abd 2024-11-14T00:27:33,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/790dc60e3a3d43aead8f442ee25d4abd, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T00:27:33,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/9e2a998618e24966abcb1665ee095191 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9e2a998618e24966abcb1665ee095191 2024-11-14T00:27:33,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9e2a998618e24966abcb1665ee095191, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T00:27:33,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/1d71d3cca089429697d99cffd0e5115a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/1d71d3cca089429697d99cffd0e5115a 2024-11-14T00:27:33,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/1d71d3cca089429697d99cffd0e5115a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T00:27:33,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 285207bf1146874945959d9159bd86b2 in 1241ms, sequenceid=13, compaction requested=false 2024-11-14T00:27:33,213 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T00:27:33,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:33,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T00:27:33,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:33,249 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T00:27:33,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/a1f11c478bc74689854e90b265d2357a is 50, key is test_row_0/A:col10/1731544052177/Put/seqid=0 2024-11-14T00:27:33,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741840_1016 (size=12001) 2024-11-14T00:27:33,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:33,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:33,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:33,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544113510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:33,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544113508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:33,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:33,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544113518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:33,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544113520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544113520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:33,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544113626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:33,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:33,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544113627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:33,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544113653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:33,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:33,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544113654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:33,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:33,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544113654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:33,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:33,716 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/a1f11c478bc74689854e90b265d2357a 2024-11-14T00:27:33,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/9238b55a2969427ab3ff4b279461ca96 is 50, key is test_row_0/B:col10/1731544052177/Put/seqid=0 2024-11-14T00:27:33,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741841_1017 (size=12001) 2024-11-14T00:27:33,802 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/9238b55a2969427ab3ff4b279461ca96 2024-11-14T00:27:33,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544113847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:33,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544113848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:33,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544113869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:33,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/167b1bb160b943bdba2a6bd82ea01718 is 50, key is test_row_0/C:col10/1731544052177/Put/seqid=0 2024-11-14T00:27:33,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544113877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:33,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544113877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:33,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:33,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:33,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741842_1018 (size=12001) 2024-11-14T00:27:33,958 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/167b1bb160b943bdba2a6bd82ea01718 2024-11-14T00:27:33,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/a1f11c478bc74689854e90b265d2357a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/a1f11c478bc74689854e90b265d2357a 2024-11-14T00:27:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T00:27:34,009 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/a1f11c478bc74689854e90b265d2357a, entries=150, sequenceid=38, filesize=11.7 K 2024-11-14T00:27:34,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/9238b55a2969427ab3ff4b279461ca96 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9238b55a2969427ab3ff4b279461ca96 2024-11-14T00:27:34,035 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9238b55a2969427ab3ff4b279461ca96, entries=150, sequenceid=38, filesize=11.7 K 2024-11-14T00:27:34,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/167b1bb160b943bdba2a6bd82ea01718 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/167b1bb160b943bdba2a6bd82ea01718 2024-11-14T00:27:34,066 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/167b1bb160b943bdba2a6bd82ea01718, entries=150, sequenceid=38, filesize=11.7 K 2024-11-14T00:27:34,068 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 285207bf1146874945959d9159bd86b2 in 819ms, sequenceid=38, compaction requested=false 2024-11-14T00:27:34,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:34,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:34,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-14T00:27:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-14T00:27:34,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T00:27:34,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2350 sec 2024-11-14T00:27:34,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.2810 sec 2024-11-14T00:27:34,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:34,184 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:27:34,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T00:27:34,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/92929719fe654569bd5e09d0ada85b16 is 50, key is test_row_0/A:col10/1731544053476/Put/seqid=0 2024-11-14T00:27:34,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741843_1019 (size=14341) 2024-11-14T00:27:34,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/92929719fe654569bd5e09d0ada85b16 2024-11-14T00:27:34,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/cfb756f62c2549e0ac4ee7c534aade10 is 50, key is test_row_0/B:col10/1731544053476/Put/seqid=0 2024-11-14T00:27:34,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741844_1020 (size=12001) 2024-11-14T00:27:34,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/cfb756f62c2549e0ac4ee7c534aade10 2024-11-14T00:27:34,366 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T00:27:34,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/394178c30f5940cc97822b7919c58607 is 50, key is test_row_0/C:col10/1731544053476/Put/seqid=0 2024-11-14T00:27:34,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544114377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:34,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:34,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:34,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544114389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:34,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544114405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:34,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:34,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544114408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:34,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741845_1021 (size=12001) 2024-11-14T00:27:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:34,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544114434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:34,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544114556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:34,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:34,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544114558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:34,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544114558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:34,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:34,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544114561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:34,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544114566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:34,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:34,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:34,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544114806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:34,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544114817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:34,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/394178c30f5940cc97822b7919c58607 2024-11-14T00:27:34,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:34,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544114829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544114830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:34,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/92929719fe654569bd5e09d0ada85b16 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/92929719fe654569bd5e09d0ada85b16 2024-11-14T00:27:34,881 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T00:27:34,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544114837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:34,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:34,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:34,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/92929719fe654569bd5e09d0ada85b16, entries=200, sequenceid=50, filesize=14.0 K 2024-11-14T00:27:34,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/cfb756f62c2549e0ac4ee7c534aade10 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/cfb756f62c2549e0ac4ee7c534aade10 2024-11-14T00:27:34,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/cfb756f62c2549e0ac4ee7c534aade10, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T00:27:34,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/394178c30f5940cc97822b7919c58607 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/394178c30f5940cc97822b7919c58607 2024-11-14T00:27:34,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/394178c30f5940cc97822b7919c58607, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T00:27:35,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 285207bf1146874945959d9159bd86b2 in 821ms, sequenceid=50, compaction requested=true 2024-11-14T00:27:35,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:35,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:27:35,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:35,013 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:35,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:27:35,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:35,013 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:35,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:27:35,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:27:35,038 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:35,039 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/B is initiating minor compaction (all files) 2024-11-14T00:27:35,040 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/B in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:35,041 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9e2a998618e24966abcb1665ee095191, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9238b55a2969427ab3ff4b279461ca96, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/cfb756f62c2549e0ac4ee7c534aade10] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=35.2 K 2024-11-14T00:27:35,052 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:35,053 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/A is initiating minor compaction (all files) 2024-11-14T00:27:35,053 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/A in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:35,053 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/790dc60e3a3d43aead8f442ee25d4abd, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/a1f11c478bc74689854e90b265d2357a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/92929719fe654569bd5e09d0ada85b16] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=37.4 K 2024-11-14T00:27:35,054 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e2a998618e24966abcb1665ee095191, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731544051908 2024-11-14T00:27:35,054 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 790dc60e3a3d43aead8f442ee25d4abd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731544051908 2024-11-14T00:27:35,061 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1f11c478bc74689854e90b265d2357a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731544052177 2024-11-14T00:27:35,061 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 9238b55a2969427ab3ff4b279461ca96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731544052177 2024-11-14T00:27:35,068 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting cfb756f62c2549e0ac4ee7c534aade10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731544053476 2024-11-14T00:27:35,068 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92929719fe654569bd5e09d0ada85b16, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731544053476 2024-11-14T00:27:35,142 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:27:35,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T00:27:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:35,174 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T00:27:35,174 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T00:27:35,181 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#A#compaction#9 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:35,182 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#B#compaction#10 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:35,189 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/80427010fc5044f199540ed10bc3c7a2 is 50, key is test_row_0/B:col10/1731544053476/Put/seqid=0 2024-11-14T00:27:35,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/491df79c0ede4643a7ea0b525da8119e is 50, key is test_row_0/A:col10/1731544054348/Put/seqid=0 2024-11-14T00:27:35,201 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T00:27:35,201 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T00:27:35,201 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T00:27:35,202 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T00:27:35,203 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T00:27:35,203 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T00:27:35,201 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/7daa78d615c24969a29c6a9a16e61592 is 50, key is test_row_0/A:col10/1731544053476/Put/seqid=0 2024-11-14T00:27:35,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741846_1022 (size=12104) 2024-11-14T00:27:35,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:35,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544115218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:35,253 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/80427010fc5044f199540ed10bc3c7a2 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/80427010fc5044f199540ed10bc3c7a2 2024-11-14T00:27:35,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:35,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544115239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:35,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:35,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544115241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:35,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:35,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544115245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:35,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:35,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544115245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:35,280 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/B of 285207bf1146874945959d9159bd86b2 into 80427010fc5044f199540ed10bc3c7a2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:35,281 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:35,283 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/B, priority=13, startTime=1731544055012; duration=0sec 2024-11-14T00:27:35,284 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:27:35,284 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:B 2024-11-14T00:27:35,284 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:35,287 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:35,287 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/C is initiating minor compaction (all files) 2024-11-14T00:27:35,287 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/C in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:35,287 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/1d71d3cca089429697d99cffd0e5115a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/167b1bb160b943bdba2a6bd82ea01718, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/394178c30f5940cc97822b7919c58607] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=35.2 K 2024-11-14T00:27:35,288 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d71d3cca089429697d99cffd0e5115a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731544051908 2024-11-14T00:27:35,294 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 167b1bb160b943bdba2a6bd82ea01718, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731544052177 2024-11-14T00:27:35,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741847_1023 (size=14341) 2024-11-14T00:27:35,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741848_1024 (size=12104) 2024-11-14T00:27:35,308 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 394178c30f5940cc97822b7919c58607, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731544053476 2024-11-14T00:27:35,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/491df79c0ede4643a7ea0b525da8119e 2024-11-14T00:27:35,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544115358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:35,378 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/7daa78d615c24969a29c6a9a16e61592 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/7daa78d615c24969a29c6a9a16e61592 2024-11-14T00:27:35,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544115377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:35,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:35,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:35,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544115377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:35,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:35,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544115378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:35,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:35,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/989f879c0758472da6ff5a260d34ff86 is 50, key is test_row_0/B:col10/1731544054348/Put/seqid=0 2024-11-14T00:27:35,402 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#C#compaction#13 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:35,403 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/06265e744d6146d796302c01a4d222a0 is 50, key is test_row_0/C:col10/1731544053476/Put/seqid=0 2024-11-14T00:27:35,404 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/A of 285207bf1146874945959d9159bd86b2 into 7daa78d615c24969a29c6a9a16e61592(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:35,404 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:35,404 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/A, priority=13, startTime=1731544055007; duration=0sec 2024-11-14T00:27:35,404 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:35,404 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:A 2024-11-14T00:27:35,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544115386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:35,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741849_1025 (size=12001) 2024-11-14T00:27:35,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/989f879c0758472da6ff5a260d34ff86 2024-11-14T00:27:35,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741850_1026 (size=12104) 2024-11-14T00:27:35,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/ebcf000e638d4f9c9b8a52f9b84e633a is 50, key is test_row_0/C:col10/1731544054348/Put/seqid=0 2024-11-14T00:27:35,540 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/06265e744d6146d796302c01a4d222a0 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/06265e744d6146d796302c01a4d222a0 2024-11-14T00:27:35,576 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/C of 285207bf1146874945959d9159bd86b2 into 06265e744d6146d796302c01a4d222a0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:35,576 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:35,576 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/C, priority=13, startTime=1731544055013; duration=0sec 2024-11-14T00:27:35,576 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:35,576 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:C 2024-11-14T00:27:35,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741851_1027 (size=12001) 2024-11-14T00:27:35,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/ebcf000e638d4f9c9b8a52f9b84e633a 2024-11-14T00:27:35,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544115577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:35,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544115592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:35,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544115597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:35,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:35,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544115599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:35,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:35,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/491df79c0ede4643a7ea0b525da8119e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/491df79c0ede4643a7ea0b525da8119e 2024-11-14T00:27:35,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544115639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:35,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/491df79c0ede4643a7ea0b525da8119e, entries=200, sequenceid=75, filesize=14.0 K 2024-11-14T00:27:35,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/989f879c0758472da6ff5a260d34ff86 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/989f879c0758472da6ff5a260d34ff86 2024-11-14T00:27:35,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/989f879c0758472da6ff5a260d34ff86, entries=150, sequenceid=75, filesize=11.7 K 2024-11-14T00:27:35,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/ebcf000e638d4f9c9b8a52f9b84e633a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ebcf000e638d4f9c9b8a52f9b84e633a 2024-11-14T00:27:35,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ebcf000e638d4f9c9b8a52f9b84e633a, entries=150, sequenceid=75, filesize=11.7 K 2024-11-14T00:27:35,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 285207bf1146874945959d9159bd86b2 in 637ms, sequenceid=75, compaction requested=false 2024-11-14T00:27:35,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:35,922 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:27:35,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T00:27:35,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/086a919a11304199beec073084346157 is 50, key is test_row_0/A:col10/1731544055226/Put/seqid=0 2024-11-14T00:27:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T00:27:36,016 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:27:36,021 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:27:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-14T00:27:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T00:27:36,025 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:27:36,028 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:27:36,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:27:36,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741852_1028 (size=12001) 2024-11-14T00:27:36,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544116069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544116073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544116100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544116093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544116101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T00:27:36,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T00:27:36,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:36,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:36,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:36,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:36,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:36,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544116216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544116216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544116231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544116226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544116236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T00:27:36,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T00:27:36,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:36,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:36,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:36,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:36,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:36,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/086a919a11304199beec073084346157 2024-11-14T00:27:36,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544116445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544116446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544116457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544116467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544116476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/f51889d9bc4046f79df5d5cf24aa3c22 is 50, key is test_row_0/B:col10/1731544055226/Put/seqid=0 2024-11-14T00:27:36,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T00:27:36,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:36,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:36,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:36,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:36,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:36,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741853_1029 (size=12001) 2024-11-14T00:27:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T00:27:36,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T00:27:36,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:36,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:36,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:36,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:36,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:36,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544116776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544116776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544116785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544116797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:36,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544116804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:36,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:36,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T00:27:36,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:36,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:36,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:36,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:36,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:36,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/f51889d9bc4046f79df5d5cf24aa3c22 2024-11-14T00:27:37,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/5160e939330043109eac4901eb7ef3e9 is 50, key is test_row_0/C:col10/1731544055226/Put/seqid=0 2024-11-14T00:27:37,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T00:27:37,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:37,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:37,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:37,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:37,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:37,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:37,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741854_1030 (size=12001) 2024-11-14T00:27:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T00:27:37,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T00:27:37,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:37,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:37,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:37,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:37,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:37,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544117296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:37,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:37,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:37,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:37,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:37,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544117307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:37,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:37,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:37,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:37,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544117317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:37,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544117307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:37,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:37,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:37,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:37,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T00:27:37,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:37,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:37,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:37,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:37,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:37,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544117329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:37,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:37,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:37,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:37,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/5160e939330043109eac4901eb7ef3e9 2024-11-14T00:27:37,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T00:27:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:37,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:37,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:37,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/086a919a11304199beec073084346157 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/086a919a11304199beec073084346157 2024-11-14T00:27:37,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/086a919a11304199beec073084346157, entries=150, sequenceid=91, filesize=11.7 K 2024-11-14T00:27:37,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/f51889d9bc4046f79df5d5cf24aa3c22 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/f51889d9bc4046f79df5d5cf24aa3c22 2024-11-14T00:27:37,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/f51889d9bc4046f79df5d5cf24aa3c22, entries=150, sequenceid=91, filesize=11.7 K 2024-11-14T00:27:37,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/5160e939330043109eac4901eb7ef3e9 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/5160e939330043109eac4901eb7ef3e9 2024-11-14T00:27:37,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/5160e939330043109eac4901eb7ef3e9, entries=150, sequenceid=91, filesize=11.7 K 2024-11-14T00:27:37,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 285207bf1146874945959d9159bd86b2 in 1705ms, sequenceid=91, compaction requested=true 2024-11-14T00:27:37,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:37,628 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:37,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:27:37,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:37,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:27:37,628 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:37,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:37,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:27:37,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:27:37,632 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:37,632 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/A is initiating minor compaction (all files) 2024-11-14T00:27:37,632 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/A in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:37,633 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/7daa78d615c24969a29c6a9a16e61592, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/491df79c0ede4643a7ea0b525da8119e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/086a919a11304199beec073084346157] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=37.5 K 2024-11-14T00:27:37,634 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:37,634 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7daa78d615c24969a29c6a9a16e61592, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731544053476 2024-11-14T00:27:37,634 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/B is initiating minor compaction (all files) 2024-11-14T00:27:37,634 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/B in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:37,634 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/80427010fc5044f199540ed10bc3c7a2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/989f879c0758472da6ff5a260d34ff86, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/f51889d9bc4046f79df5d5cf24aa3c22] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=35.3 K 2024-11-14T00:27:37,636 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 80427010fc5044f199540ed10bc3c7a2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731544053476 2024-11-14T00:27:37,636 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 491df79c0ede4643a7ea0b525da8119e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731544054348 2024-11-14T00:27:37,637 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 989f879c0758472da6ff5a260d34ff86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731544054348 2024-11-14T00:27:37,638 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 086a919a11304199beec073084346157, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731544055183 2024-11-14T00:27:37,639 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting f51889d9bc4046f79df5d5cf24aa3c22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731544055183 2024-11-14T00:27:37,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T00:27:37,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:37,665 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T00:27:37,676 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:37,677 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/9c90db9c64a144f9a46b44bea09a5fff is 50, key is test_row_0/B:col10/1731544055226/Put/seqid=0 2024-11-14T00:27:37,682 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#A#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:37,683 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/0ced9a1a4db74d968dda0f7045450492 is 50, key is test_row_0/A:col10/1731544055226/Put/seqid=0 2024-11-14T00:27:37,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/0d8c4d88034f4d438dabc9e100af1a9b is 50, key is test_row_0/A:col10/1731544056065/Put/seqid=0 2024-11-14T00:27:37,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741855_1031 (size=12207) 2024-11-14T00:27:37,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741856_1032 (size=12207) 2024-11-14T00:27:37,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741857_1033 (size=12001) 2024-11-14T00:27:37,797 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/0d8c4d88034f4d438dabc9e100af1a9b 2024-11-14T00:27:37,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/970cd479196e4c48be8a09704e6b97bb is 50, key is test_row_0/B:col10/1731544056065/Put/seqid=0 2024-11-14T00:27:37,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741858_1034 (size=12001) 2024-11-14T00:27:38,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T00:27:38,216 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/0ced9a1a4db74d968dda0f7045450492 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0ced9a1a4db74d968dda0f7045450492 2024-11-14T00:27:38,242 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/9c90db9c64a144f9a46b44bea09a5fff as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9c90db9c64a144f9a46b44bea09a5fff 2024-11-14T00:27:38,263 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/B of 285207bf1146874945959d9159bd86b2 into 9c90db9c64a144f9a46b44bea09a5fff(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:38,264 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:38,264 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/B, priority=13, startTime=1731544057628; duration=0sec 2024-11-14T00:27:38,265 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:27:38,265 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:B 2024-11-14T00:27:38,265 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:38,265 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/A of 285207bf1146874945959d9159bd86b2 into 0ced9a1a4db74d968dda0f7045450492(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:38,265 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:38,265 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/A, priority=13, startTime=1731544057628; duration=0sec 2024-11-14T00:27:38,266 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:38,266 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:A 2024-11-14T00:27:38,269 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:38,270 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/C is initiating minor compaction (all files) 2024-11-14T00:27:38,270 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/C in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:38,270 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/06265e744d6146d796302c01a4d222a0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ebcf000e638d4f9c9b8a52f9b84e633a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/5160e939330043109eac4901eb7ef3e9] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=35.3 K 2024-11-14T00:27:38,271 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 06265e744d6146d796302c01a4d222a0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731544053476 2024-11-14T00:27:38,273 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting ebcf000e638d4f9c9b8a52f9b84e633a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731544054348 2024-11-14T00:27:38,275 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 5160e939330043109eac4901eb7ef3e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731544055183 2024-11-14T00:27:38,278 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/970cd479196e4c48be8a09704e6b97bb 2024-11-14T00:27:38,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/35d72ce9416644bead9b38f0dafd044d is 50, key is test_row_0/C:col10/1731544056065/Put/seqid=0 2024-11-14T00:27:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:38,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:38,325 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:38,326 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/3026c3739a9045a89fcc1058398dff75 is 50, key is test_row_0/C:col10/1731544055226/Put/seqid=0 2024-11-14T00:27:38,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741859_1035 (size=12001) 2024-11-14T00:27:38,383 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/35d72ce9416644bead9b38f0dafd044d 2024-11-14T00:27:38,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:38,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/0d8c4d88034f4d438dabc9e100af1a9b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0d8c4d88034f4d438dabc9e100af1a9b 2024-11-14T00:27:38,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544118365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:38,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:38,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544118378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741860_1036 (size=12207) 2024-11-14T00:27:38,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:38,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:38,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544118382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:38,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:38,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544118383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:38,415 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_2' in 285207bf1146874945959d9159bd86b2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2320 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:27:38,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:38,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544118400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:38,438 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0d8c4d88034f4d438dabc9e100af1a9b, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T00:27:38,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/970cd479196e4c48be8a09704e6b97bb as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/970cd479196e4c48be8a09704e6b97bb 2024-11-14T00:27:38,454 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/3026c3739a9045a89fcc1058398dff75 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/3026c3739a9045a89fcc1058398dff75 2024-11-14T00:27:38,472 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/970cd479196e4c48be8a09704e6b97bb, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T00:27:38,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/35d72ce9416644bead9b38f0dafd044d as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/35d72ce9416644bead9b38f0dafd044d 2024-11-14T00:27:38,489 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/C of 285207bf1146874945959d9159bd86b2 into 3026c3739a9045a89fcc1058398dff75(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:38,490 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:38,490 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/C, priority=13, startTime=1731544057628; duration=0sec 2024-11-14T00:27:38,492 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:38,492 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:C 2024-11-14T00:27:38,517 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/35d72ce9416644bead9b38f0dafd044d, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T00:27:38,525 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 285207bf1146874945959d9159bd86b2 in 859ms, sequenceid=114, compaction requested=false 2024-11-14T00:27:38,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:38,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:38,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-14T00:27:38,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-14T00:27:38,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:38,538 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:27:38,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T00:27:38,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-14T00:27:38,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5090 sec 2024-11-14T00:27:38,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.5310 sec 2024-11-14T00:27:38,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/0e7940092e2c4c41adec1d1aa1c0012c is 50, key is test_row_0/A:col10/1731544058538/Put/seqid=0 2024-11-14T00:27:38,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741861_1037 (size=12151) 2024-11-14T00:27:38,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544118805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:38,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544118817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:38,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544118817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:38,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:38,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544118819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:38,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544118935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:38,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544118936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:38,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544118936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:38,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544118936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:38,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:38,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/0e7940092e2c4c41adec1d1aa1c0012c 2024-11-14T00:27:39,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/b5a92b6858914b428a25ff2275e74c5c is 50, key is test_row_0/B:col10/1731544058538/Put/seqid=0 2024-11-14T00:27:39,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741862_1038 (size=12151) 2024-11-14T00:27:39,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/b5a92b6858914b428a25ff2275e74c5c 2024-11-14T00:27:39,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544119155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/f9a7d44607184ddabddcd1b6d251aee5 is 50, key is test_row_0/C:col10/1731544058538/Put/seqid=0 2024-11-14T00:27:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544119156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:39,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544119165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:39,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544119166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741863_1039 (size=12151) 2024-11-14T00:27:39,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/f9a7d44607184ddabddcd1b6d251aee5 2024-11-14T00:27:39,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/0e7940092e2c4c41adec1d1aa1c0012c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0e7940092e2c4c41adec1d1aa1c0012c 2024-11-14T00:27:39,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0e7940092e2c4c41adec1d1aa1c0012c, entries=150, sequenceid=133, filesize=11.9 K 2024-11-14T00:27:39,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/b5a92b6858914b428a25ff2275e74c5c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/b5a92b6858914b428a25ff2275e74c5c 2024-11-14T00:27:39,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/b5a92b6858914b428a25ff2275e74c5c, entries=150, sequenceid=133, filesize=11.9 K 2024-11-14T00:27:39,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/f9a7d44607184ddabddcd1b6d251aee5 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/f9a7d44607184ddabddcd1b6d251aee5 2024-11-14T00:27:39,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/f9a7d44607184ddabddcd1b6d251aee5, entries=150, sequenceid=133, filesize=11.9 K 2024-11-14T00:27:39,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=127.47 KB/130530 for 285207bf1146874945959d9159bd86b2 in 815ms, sequenceid=133, compaction requested=true 2024-11-14T00:27:39,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:39,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:27:39,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:39,353 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:39,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:27:39,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:27:39,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:27:39,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:27:39,355 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:39,356 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:39,356 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/A is initiating minor compaction (all files) 2024-11-14T00:27:39,356 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/A in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:39,356 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0ced9a1a4db74d968dda0f7045450492, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0d8c4d88034f4d438dabc9e100af1a9b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0e7940092e2c4c41adec1d1aa1c0012c] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=35.5 K 2024-11-14T00:27:39,357 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:39,357 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/B is initiating minor compaction (all files) 2024-11-14T00:27:39,357 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/B in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:39,357 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9c90db9c64a144f9a46b44bea09a5fff, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/970cd479196e4c48be8a09704e6b97bb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/b5a92b6858914b428a25ff2275e74c5c] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=35.5 K 2024-11-14T00:27:39,358 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ced9a1a4db74d968dda0f7045450492, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731544055183 2024-11-14T00:27:39,359 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c90db9c64a144f9a46b44bea09a5fff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731544055183 2024-11-14T00:27:39,359 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d8c4d88034f4d438dabc9e100af1a9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731544056065 2024-11-14T00:27:39,360 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 970cd479196e4c48be8a09704e6b97bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731544056065 2024-11-14T00:27:39,360 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e7940092e2c4c41adec1d1aa1c0012c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731544058532 2024-11-14T00:27:39,361 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting b5a92b6858914b428a25ff2275e74c5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731544058532 2024-11-14T00:27:39,401 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#B#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:39,402 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/1e8e8196a3544aa09b4f3ac96df52e36 is 50, key is test_row_0/B:col10/1731544058538/Put/seqid=0 2024-11-14T00:27:39,411 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:39,412 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/8ac78c1b0e524160840c2dae49ddf7a2 is 50, key is test_row_0/A:col10/1731544058538/Put/seqid=0 2024-11-14T00:27:39,471 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:27:39,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T00:27:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:39,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/5320e84f839d4af8b41ec4048ab53962 is 50, key is test_row_0/A:col10/1731544058804/Put/seqid=0 2024-11-14T00:27:39,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741864_1040 (size=12459) 2024-11-14T00:27:39,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741865_1041 (size=12459) 2024-11-14T00:27:39,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741866_1042 (size=14541) 2024-11-14T00:27:39,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/5320e84f839d4af8b41ec4048ab53962 2024-11-14T00:27:39,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:39,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544119538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544119550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:39,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,573 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/1e8e8196a3544aa09b4f3ac96df52e36 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/1e8e8196a3544aa09b4f3ac96df52e36 2024-11-14T00:27:39,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/cfb2c42fe91e4e399be52c252b80a597 is 50, key is test_row_0/B:col10/1731544058804/Put/seqid=0 2024-11-14T00:27:39,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544119569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:39,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544119582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:39,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,597 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/B of 285207bf1146874945959d9159bd86b2 into 1e8e8196a3544aa09b4f3ac96df52e36(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:39,598 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:39,598 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/B, priority=13, startTime=1731544059353; duration=0sec 2024-11-14T00:27:39,598 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:27:39,598 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:B 2024-11-14T00:27:39,598 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:39,600 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:39,601 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/C is initiating minor compaction (all files) 2024-11-14T00:27:39,601 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/C in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:39,601 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/3026c3739a9045a89fcc1058398dff75, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/35d72ce9416644bead9b38f0dafd044d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/f9a7d44607184ddabddcd1b6d251aee5] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=35.5 K 2024-11-14T00:27:39,604 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 3026c3739a9045a89fcc1058398dff75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731544055183 2024-11-14T00:27:39,605 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 35d72ce9416644bead9b38f0dafd044d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731544056065 2024-11-14T00:27:39,607 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting f9a7d44607184ddabddcd1b6d251aee5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731544058532 2024-11-14T00:27:39,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741867_1043 (size=12151) 2024-11-14T00:27:39,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/cfb2c42fe91e4e399be52c252b80a597 2024-11-14T00:27:39,648 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#C#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:39,649 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/a1480ac9a9ef47e4bb10841e471aa854 is 50, key is test_row_0/C:col10/1731544058538/Put/seqid=0 2024-11-14T00:27:39,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/b7a6d015598f45bca00e81fbcce5355e is 50, key is test_row_0/C:col10/1731544058804/Put/seqid=0 2024-11-14T00:27:39,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544119666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544119675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:39,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:39,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544119705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:39,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:39,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544119708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:39,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741869_1045 (size=12459) 2024-11-14T00:27:39,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741868_1044 (size=12151) 2024-11-14T00:27:39,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544119887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544119897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544119915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544119926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:39,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:39,955 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/8ac78c1b0e524160840c2dae49ddf7a2 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/8ac78c1b0e524160840c2dae49ddf7a2 2024-11-14T00:27:39,968 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/A of 285207bf1146874945959d9159bd86b2 into 8ac78c1b0e524160840c2dae49ddf7a2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:39,968 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:39,968 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/A, priority=13, startTime=1731544059353; duration=0sec 2024-11-14T00:27:39,968 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:39,968 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:A 2024-11-14T00:27:40,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/b7a6d015598f45bca00e81fbcce5355e 2024-11-14T00:27:40,144 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/a1480ac9a9ef47e4bb10841e471aa854 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/a1480ac9a9ef47e4bb10841e471aa854 2024-11-14T00:27:40,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/5320e84f839d4af8b41ec4048ab53962 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/5320e84f839d4af8b41ec4048ab53962 2024-11-14T00:27:40,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/5320e84f839d4af8b41ec4048ab53962, entries=200, sequenceid=156, filesize=14.2 K 2024-11-14T00:27:40,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/cfb2c42fe91e4e399be52c252b80a597 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/cfb2c42fe91e4e399be52c252b80a597 2024-11-14T00:27:40,165 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/C of 285207bf1146874945959d9159bd86b2 into a1480ac9a9ef47e4bb10841e471aa854(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:40,165 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:40,165 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/C, priority=13, startTime=1731544059353; duration=0sec 2024-11-14T00:27:40,165 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:40,166 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:C 2024-11-14T00:27:40,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/cfb2c42fe91e4e399be52c252b80a597, entries=150, sequenceid=156, filesize=11.9 K 2024-11-14T00:27:40,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/b7a6d015598f45bca00e81fbcce5355e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b7a6d015598f45bca00e81fbcce5355e 2024-11-14T00:27:40,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b7a6d015598f45bca00e81fbcce5355e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-14T00:27:40,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 285207bf1146874945959d9159bd86b2 in 712ms, sequenceid=156, compaction requested=false 2024-11-14T00:27:40,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:40,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T00:27:40,197 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:27:40,199 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:27:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-14T00:27:40,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T00:27:40,205 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:27:40,206 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:27:40,207 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:27:40,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:40,224 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:27:40,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T00:27:40,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/2bd9e3f3ea6c4f6ebd0432cbb1486fad is 50, key is test_row_0/A:col10/1731544060215/Put/seqid=0 2024-11-14T00:27:40,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741870_1046 (size=14541) 2024-11-14T00:27:40,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/2bd9e3f3ea6c4f6ebd0432cbb1486fad 2024-11-14T00:27:40,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/c4c8ea6bf3ca43bb8b2bd191292c08a6 is 50, key is test_row_0/B:col10/1731544060215/Put/seqid=0 2024-11-14T00:27:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T00:27:40,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544120309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:40,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544120321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:40,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:40,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:40,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544120326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:40,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544120316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:40,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741871_1047 (size=12151) 2024-11-14T00:27:40,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T00:27:40,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:40,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:40,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:40,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:40,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:40,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:40,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:40,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544120436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:40,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544120436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:40,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544120436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:40,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544120436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544120435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:40,449 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_2' in 285207bf1146874945959d9159bd86b2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4356 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:27:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:40,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T00:27:40,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:40,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:40,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:40,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:40,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:40,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T00:27:40,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544120655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544120656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544120656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:40,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:40,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:40,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544120656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:40,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:40,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T00:27:40,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:40,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:40,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:40,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:40,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:40,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/c4c8ea6bf3ca43bb8b2bd191292c08a6 2024-11-14T00:27:40,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/b9c3048a8f4545e4be97a8bd239e0670 is 50, key is test_row_0/C:col10/1731544060215/Put/seqid=0 2024-11-14T00:27:40,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741872_1048 (size=12151) 2024-11-14T00:27:40,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/b9c3048a8f4545e4be97a8bd239e0670 2024-11-14T00:27:40,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/2bd9e3f3ea6c4f6ebd0432cbb1486fad as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/2bd9e3f3ea6c4f6ebd0432cbb1486fad 2024-11-14T00:27:40,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/2bd9e3f3ea6c4f6ebd0432cbb1486fad, entries=200, sequenceid=175, filesize=14.2 K 2024-11-14T00:27:40,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/c4c8ea6bf3ca43bb8b2bd191292c08a6 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/c4c8ea6bf3ca43bb8b2bd191292c08a6 2024-11-14T00:27:40,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/c4c8ea6bf3ca43bb8b2bd191292c08a6, entries=150, sequenceid=175, filesize=11.9 K 2024-11-14T00:27:40,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/b9c3048a8f4545e4be97a8bd239e0670 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b9c3048a8f4545e4be97a8bd239e0670 2024-11-14T00:27:40,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b9c3048a8f4545e4be97a8bd239e0670, entries=150, sequenceid=175, filesize=11.9 K 2024-11-14T00:27:40,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 285207bf1146874945959d9159bd86b2 in 605ms, sequenceid=175, compaction requested=true 2024-11-14T00:27:40,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:40,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:27:40,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:40,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:27:40,829 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:40,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:27:40,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:27:40,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:27:40,829 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:40,832 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:40,833 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/B is initiating minor compaction (all files) 2024-11-14T00:27:40,833 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/B in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:40,833 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/1e8e8196a3544aa09b4f3ac96df52e36, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/cfb2c42fe91e4e399be52c252b80a597, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/c4c8ea6bf3ca43bb8b2bd191292c08a6] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=35.9 K 2024-11-14T00:27:40,833 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41541 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:40,833 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/A is initiating minor compaction (all files) 2024-11-14T00:27:40,833 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/A in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:40,834 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/8ac78c1b0e524160840c2dae49ddf7a2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/5320e84f839d4af8b41ec4048ab53962, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/2bd9e3f3ea6c4f6ebd0432cbb1486fad] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=40.6 K 2024-11-14T00:27:40,835 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e8e8196a3544aa09b4f3ac96df52e36, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731544058532 2024-11-14T00:27:40,835 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ac78c1b0e524160840c2dae49ddf7a2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731544058532 2024-11-14T00:27:40,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T00:27:40,836 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting cfb2c42fe91e4e399be52c252b80a597, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731544058802 2024-11-14T00:27:40,836 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5320e84f839d4af8b41ec4048ab53962, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731544058799 2024-11-14T00:27:40,837 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c4c8ea6bf3ca43bb8b2bd191292c08a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731544059563 2024-11-14T00:27:40,838 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bd9e3f3ea6c4f6ebd0432cbb1486fad, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731544059551 2024-11-14T00:27:40,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T00:27:40,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:40,842 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-14T00:27:40,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/3856a4c6994141e9a9f21ec39f7682f5 is 50, key is test_row_0/A:col10/1731544060321/Put/seqid=0 2024-11-14T00:27:40,871 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:40,872 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/76acba3d34824f6ebddfccfe01308bd8 is 50, key is test_row_0/B:col10/1731544060215/Put/seqid=0 2024-11-14T00:27:40,877 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#A#compaction#38 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:40,878 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/1740c2734157494a99b3a4f397c02209 is 50, key is test_row_0/A:col10/1731544060215/Put/seqid=0 2024-11-14T00:27:40,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741873_1049 (size=12151) 2024-11-14T00:27:40,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741874_1050 (size=12561) 2024-11-14T00:27:40,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:40,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741875_1051 (size=12561) 2024-11-14T00:27:41,005 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/1740c2734157494a99b3a4f397c02209 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/1740c2734157494a99b3a4f397c02209 2024-11-14T00:27:41,025 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/A of 285207bf1146874945959d9159bd86b2 into 1740c2734157494a99b3a4f397c02209(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:41,025 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:41,025 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/A, priority=13, startTime=1731544060829; duration=0sec 2024-11-14T00:27:41,025 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:27:41,025 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:A 2024-11-14T00:27:41,025 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:41,027 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:41,027 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/C is initiating minor compaction (all files) 2024-11-14T00:27:41,027 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/C in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:41,027 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/a1480ac9a9ef47e4bb10841e471aa854, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b7a6d015598f45bca00e81fbcce5355e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b9c3048a8f4545e4be97a8bd239e0670] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=35.9 K 2024-11-14T00:27:41,030 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1480ac9a9ef47e4bb10841e471aa854, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731544058532 2024-11-14T00:27:41,030 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7a6d015598f45bca00e81fbcce5355e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731544058802 2024-11-14T00:27:41,031 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9c3048a8f4545e4be97a8bd239e0670, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731544059563 2024-11-14T00:27:41,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544121054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:41,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544121056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:41,074 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#C#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:41,074 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/fb15d6f96893448d976998b1a8023543 is 50, key is test_row_0/C:col10/1731544060215/Put/seqid=0 2024-11-14T00:27:41,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:41,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544121064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:41,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:41,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544121076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:41,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:41,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741876_1052 (size=12561) 2024-11-14T00:27:41,133 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/fb15d6f96893448d976998b1a8023543 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/fb15d6f96893448d976998b1a8023543 2024-11-14T00:27:41,149 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/C of 285207bf1146874945959d9159bd86b2 into fb15d6f96893448d976998b1a8023543(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:41,150 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:41,150 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/C, priority=13, startTime=1731544060829; duration=0sec 2024-11-14T00:27:41,151 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:41,151 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:C 2024-11-14T00:27:41,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544121176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544121176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:41,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:41,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544121186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:41,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:41,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544121186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:41,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:41,332 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/3856a4c6994141e9a9f21ec39f7682f5 2024-11-14T00:27:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T00:27:41,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/5359ee3545d649979f8e19c86adc9dd8 is 50, key is test_row_0/B:col10/1731544060321/Put/seqid=0 2024-11-14T00:27:41,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741877_1053 (size=12151) 2024-11-14T00:27:41,365 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/5359ee3545d649979f8e19c86adc9dd8 2024-11-14T00:27:41,392 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/76acba3d34824f6ebddfccfe01308bd8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/76acba3d34824f6ebddfccfe01308bd8 2024-11-14T00:27:41,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544121386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544121386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:41,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/066fc4dbd099496cbb4d6c51f6052ae9 is 50, key is test_row_0/C:col10/1731544060321/Put/seqid=0 2024-11-14T00:27:41,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544121400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:41,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544121406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:41,414 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/B of 285207bf1146874945959d9159bd86b2 into 76acba3d34824f6ebddfccfe01308bd8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:41,414 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:41,414 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/B, priority=13, startTime=1731544060829; duration=0sec 2024-11-14T00:27:41,416 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:41,416 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:B 2024-11-14T00:27:41,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741878_1054 (size=12151) 2024-11-14T00:27:41,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544121709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:41,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544121709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:41,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544121719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:41,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:41,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544121720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:41,840 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/066fc4dbd099496cbb4d6c51f6052ae9 2024-11-14T00:27:41,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/3856a4c6994141e9a9f21ec39f7682f5 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/3856a4c6994141e9a9f21ec39f7682f5 2024-11-14T00:27:41,877 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/3856a4c6994141e9a9f21ec39f7682f5, entries=150, sequenceid=195, filesize=11.9 K 2024-11-14T00:27:41,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/5359ee3545d649979f8e19c86adc9dd8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/5359ee3545d649979f8e19c86adc9dd8 2024-11-14T00:27:41,897 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/5359ee3545d649979f8e19c86adc9dd8, entries=150, sequenceid=195, filesize=11.9 K 2024-11-14T00:27:41,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/066fc4dbd099496cbb4d6c51f6052ae9 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/066fc4dbd099496cbb4d6c51f6052ae9 2024-11-14T00:27:41,920 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/066fc4dbd099496cbb4d6c51f6052ae9, entries=150, sequenceid=195, filesize=11.9 K 2024-11-14T00:27:41,921 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 285207bf1146874945959d9159bd86b2 in 1079ms, sequenceid=195, compaction requested=false 2024-11-14T00:27:41,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:41,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:41,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-14T00:27:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-14T00:27:41,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-14T00:27:41,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7170 sec 2024-11-14T00:27:41,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.7290 sec 2024-11-14T00:27:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:42,227 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:27:42,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T00:27:42,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/8570d31b51634923839a37b3239d9970 is 50, key is test_row_0/A:col10/1731544061014/Put/seqid=0 2024-11-14T00:27:42,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741879_1055 (size=14541) 2024-11-14T00:27:42,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544122263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:42,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:42,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544122264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:42,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:42,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544122265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:42,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544122272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T00:27:42,356 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:27:42,356 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:27:42,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-14T00:27:42,360 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:27:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T00:27:42,362 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:27:42,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:27:42,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544122376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:42,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544122376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:42,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544122386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:42,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:42,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544122387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T00:27:42,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T00:27:42,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:42,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:42,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:42,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:42,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:42,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544122588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:42,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544122589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:42,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:42,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544122608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:42,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544122610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:42,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/8570d31b51634923839a37b3239d9970 2024-11-14T00:27:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T00:27:42,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T00:27:42,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:42,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:42,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:42,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:42,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:42,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/2b58d3515b514ae6b7a4bcc7f6180d65 is 50, key is test_row_0/B:col10/1731544061014/Put/seqid=0 2024-11-14T00:27:42,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741880_1056 (size=12151) 2024-11-14T00:27:42,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T00:27:42,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:42,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:42,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:42,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:42,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:42,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544122907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:42,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:42,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544122917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:42,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:42,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544122916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:42,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:42,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544122925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:42,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T00:27:43,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T00:27:43,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:43,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:43,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:43,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:43,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:43,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/2b58d3515b514ae6b7a4bcc7f6180d65 2024-11-14T00:27:43,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/6b85b0806ed34988af7faad5c94d88e8 is 50, key is test_row_0/C:col10/1731544061014/Put/seqid=0 2024-11-14T00:27:43,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T00:27:43,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741881_1057 (size=12151) 2024-11-14T00:27:43,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/6b85b0806ed34988af7faad5c94d88e8 2024-11-14T00:27:43,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:43,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:43,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:43,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:43,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:43,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/8570d31b51634923839a37b3239d9970 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/8570d31b51634923839a37b3239d9970 2024-11-14T00:27:43,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/8570d31b51634923839a37b3239d9970, entries=200, sequenceid=216, filesize=14.2 K 2024-11-14T00:27:43,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/2b58d3515b514ae6b7a4bcc7f6180d65 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/2b58d3515b514ae6b7a4bcc7f6180d65 2024-11-14T00:27:43,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/2b58d3515b514ae6b7a4bcc7f6180d65, entries=150, sequenceid=216, filesize=11.9 K 2024-11-14T00:27:43,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/6b85b0806ed34988af7faad5c94d88e8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/6b85b0806ed34988af7faad5c94d88e8 2024-11-14T00:27:43,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/6b85b0806ed34988af7faad5c94d88e8, entries=150, sequenceid=216, filesize=11.9 K 2024-11-14T00:27:43,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 285207bf1146874945959d9159bd86b2 in 1020ms, sequenceid=216, compaction requested=true 2024-11-14T00:27:43,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:43,248 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:43,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:27:43,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:43,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:27:43,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:27:43,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:27:43,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:27:43,250 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:43,250 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/A is initiating minor compaction (all files) 2024-11-14T00:27:43,250 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/A in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:43,250 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/1740c2734157494a99b3a4f397c02209, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/3856a4c6994141e9a9f21ec39f7682f5, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/8570d31b51634923839a37b3239d9970] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=38.3 K 2024-11-14T00:27:43,251 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1740c2734157494a99b3a4f397c02209, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731544059563 2024-11-14T00:27:43,252 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3856a4c6994141e9a9f21ec39f7682f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731544060254 2024-11-14T00:27:43,252 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:43,254 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8570d31b51634923839a37b3239d9970, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731544061014 2024-11-14T00:27:43,261 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:43,262 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/B is initiating minor compaction (all files) 2024-11-14T00:27:43,262 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/B in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:43,262 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/76acba3d34824f6ebddfccfe01308bd8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/5359ee3545d649979f8e19c86adc9dd8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/2b58d3515b514ae6b7a4bcc7f6180d65] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=36.0 K 2024-11-14T00:27:43,263 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 76acba3d34824f6ebddfccfe01308bd8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731544059563 2024-11-14T00:27:43,264 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 5359ee3545d649979f8e19c86adc9dd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731544060254 2024-11-14T00:27:43,267 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b58d3515b514ae6b7a4bcc7f6180d65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731544061014 2024-11-14T00:27:43,282 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#A#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:43,282 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/237df47a958f4c3d82af6abd25f06bfe is 50, key is test_row_0/A:col10/1731544061014/Put/seqid=0 2024-11-14T00:27:43,302 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#B#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:43,303 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/79983163999047769379c6a67fba9eec is 50, key is test_row_0/B:col10/1731544061014/Put/seqid=0 2024-11-14T00:27:43,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T00:27:43,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741882_1058 (size=12663) 2024-11-14T00:27:43,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:43,328 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-14T00:27:43,342 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/237df47a958f4c3d82af6abd25f06bfe as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/237df47a958f4c3d82af6abd25f06bfe 2024-11-14T00:27:43,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/7b6c672bcc2241a89160d3d721801444 is 50, key is test_row_0/A:col10/1731544062268/Put/seqid=0 2024-11-14T00:27:43,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741883_1059 (size=12663) 2024-11-14T00:27:43,370 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/79983163999047769379c6a67fba9eec as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/79983163999047769379c6a67fba9eec 2024-11-14T00:27:43,379 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/A of 285207bf1146874945959d9159bd86b2 into 237df47a958f4c3d82af6abd25f06bfe(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:43,379 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:43,379 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/A, priority=13, startTime=1731544063248; duration=0sec 2024-11-14T00:27:43,379 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:27:43,379 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:A 2024-11-14T00:27:43,379 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:43,382 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:43,383 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/C is initiating minor compaction (all files) 2024-11-14T00:27:43,383 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/C in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:43,383 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/fb15d6f96893448d976998b1a8023543, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/066fc4dbd099496cbb4d6c51f6052ae9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/6b85b0806ed34988af7faad5c94d88e8] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=36.0 K 2024-11-14T00:27:43,384 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb15d6f96893448d976998b1a8023543, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731544059563 2024-11-14T00:27:43,384 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 066fc4dbd099496cbb4d6c51f6052ae9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731544060254 2024-11-14T00:27:43,385 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/B of 285207bf1146874945959d9159bd86b2 into 79983163999047769379c6a67fba9eec(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:43,385 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:43,385 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/B, priority=13, startTime=1731544063248; duration=0sec 2024-11-14T00:27:43,385 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b85b0806ed34988af7faad5c94d88e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731544061014 2024-11-14T00:27:43,388 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:43,388 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:B 2024-11-14T00:27:43,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741884_1060 (size=12151) 2024-11-14T00:27:43,415 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#C#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:43,415 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/35997686d672440f94e8fa01c8ce3378 is 50, key is test_row_0/C:col10/1731544061014/Put/seqid=0 2024-11-14T00:27:43,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741885_1061 (size=12663) 2024-11-14T00:27:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:43,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:43,455 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/35997686d672440f94e8fa01c8ce3378 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/35997686d672440f94e8fa01c8ce3378 2024-11-14T00:27:43,473 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/C of 285207bf1146874945959d9159bd86b2 into 35997686d672440f94e8fa01c8ce3378(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:43,473 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:43,473 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/C, priority=13, startTime=1731544063248; duration=0sec 2024-11-14T00:27:43,473 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:43,473 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:C 2024-11-14T00:27:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T00:27:43,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:43,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544123502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:43,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:43,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:43,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544123504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:43,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:43,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:43,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544123509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:43,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:43,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544123511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:43,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:43,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544123616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:43,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544123625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:43,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544123625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544123625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:43,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:43,793 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/7b6c672bcc2241a89160d3d721801444 2024-11-14T00:27:43,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/d2861f6f996f438cb7bbfae84ddd9e30 is 50, key is test_row_0/B:col10/1731544062268/Put/seqid=0 2024-11-14T00:27:43,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544123825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:43,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544123836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:43,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544123836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:43,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:43,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544123836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:43,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741886_1062 (size=12151) 2024-11-14T00:27:43,847 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/d2861f6f996f438cb7bbfae84ddd9e30 2024-11-14T00:27:43,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/a3c1275f29284116b91e94b8695667ae is 50, key is test_row_0/C:col10/1731544062268/Put/seqid=0 2024-11-14T00:27:43,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741887_1063 (size=12151) 2024-11-14T00:27:43,927 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/a3c1275f29284116b91e94b8695667ae 2024-11-14T00:27:43,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/7b6c672bcc2241a89160d3d721801444 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/7b6c672bcc2241a89160d3d721801444 2024-11-14T00:27:43,954 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/7b6c672bcc2241a89160d3d721801444, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T00:27:43,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/d2861f6f996f438cb7bbfae84ddd9e30 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/d2861f6f996f438cb7bbfae84ddd9e30 2024-11-14T00:27:43,969 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/d2861f6f996f438cb7bbfae84ddd9e30, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T00:27:43,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/a3c1275f29284116b91e94b8695667ae as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/a3c1275f29284116b91e94b8695667ae 2024-11-14T00:27:44,005 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/a3c1275f29284116b91e94b8695667ae, entries=150, sequenceid=234, filesize=11.9 K 2024-11-14T00:27:44,007 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 285207bf1146874945959d9159bd86b2 in 679ms, sequenceid=234, compaction requested=false 2024-11-14T00:27:44,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:44,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:44,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-14T00:27:44,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-14T00:27:44,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T00:27:44,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6870 sec 2024-11-14T00:27:44,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.7180 sec 2024-11-14T00:27:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:44,148 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:27:44,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-14T00:27:44,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/55659d32c36347f7a78fa3f359b3c70e is 50, key is test_row_0/A:col10/1731544063497/Put/seqid=0 2024-11-14T00:27:44,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741888_1064 (size=14541) 2024-11-14T00:27:44,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/55659d32c36347f7a78fa3f359b3c70e 2024-11-14T00:27:44,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544124205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544124206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544124213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544124217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/c657a8ec907c418ba117a26864c5669c is 50, key is test_row_0/B:col10/1731544063497/Put/seqid=0 2024-11-14T00:27:44,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741889_1065 (size=12151) 2024-11-14T00:27:44,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544124325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544124325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544124336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544124336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58754 deadline: 1731544124456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,461 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_2' in 285207bf1146874945959d9159bd86b2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8368 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:27:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T00:27:44,510 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:27:44,511 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:27:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-14T00:27:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T00:27:44,522 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:27:44,524 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:27:44,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:27:44,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544124536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544124536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544124545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544124556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T00:27:44,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/c657a8ec907c418ba117a26864c5669c 2024-11-14T00:27:44,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T00:27:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:44,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:44,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:44,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/b10dbfa6055a48b18c77a99f10d34ca8 is 50, key is test_row_0/C:col10/1731544063497/Put/seqid=0 2024-11-14T00:27:44,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741890_1066 (size=12151) 2024-11-14T00:27:44,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T00:27:44,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T00:27:44,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:44,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:44,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:44,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:44,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:44,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:44,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544124845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544124855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544124865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544124865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:44,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:44,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T00:27:44,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:44,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:44,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:44,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:44,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:44,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:45,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/b10dbfa6055a48b18c77a99f10d34ca8 2024-11-14T00:27:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T00:27:45,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T00:27:45,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:45,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:45,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:45,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:45,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:45,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/55659d32c36347f7a78fa3f359b3c70e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/55659d32c36347f7a78fa3f359b3c70e 2024-11-14T00:27:45,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/55659d32c36347f7a78fa3f359b3c70e, entries=200, sequenceid=256, filesize=14.2 K 2024-11-14T00:27:45,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/c657a8ec907c418ba117a26864c5669c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/c657a8ec907c418ba117a26864c5669c 2024-11-14T00:27:45,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/c657a8ec907c418ba117a26864c5669c, entries=150, sequenceid=256, filesize=11.9 K 2024-11-14T00:27:45,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/b10dbfa6055a48b18c77a99f10d34ca8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b10dbfa6055a48b18c77a99f10d34ca8 2024-11-14T00:27:45,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b10dbfa6055a48b18c77a99f10d34ca8, entries=150, sequenceid=256, filesize=11.9 K 2024-11-14T00:27:45,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 285207bf1146874945959d9159bd86b2 in 1056ms, sequenceid=256, compaction requested=true 2024-11-14T00:27:45,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:45,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:27:45,205 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:45,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:45,205 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:45,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:27:45,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:45,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:27:45,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:27:45,209 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:45,209 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/B is initiating minor compaction (all files) 2024-11-14T00:27:45,209 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/B in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:45,210 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/79983163999047769379c6a67fba9eec, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/d2861f6f996f438cb7bbfae84ddd9e30, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/c657a8ec907c418ba117a26864c5669c] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=36.1 K 2024-11-14T00:27:45,210 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:45,210 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/A is initiating minor compaction (all files) 2024-11-14T00:27:45,210 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/A in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:45,210 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/237df47a958f4c3d82af6abd25f06bfe, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/7b6c672bcc2241a89160d3d721801444, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/55659d32c36347f7a78fa3f359b3c70e] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=38.4 K 2024-11-14T00:27:45,213 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 79983163999047769379c6a67fba9eec, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731544061014 2024-11-14T00:27:45,214 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 237df47a958f4c3d82af6abd25f06bfe, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731544061014 2024-11-14T00:27:45,214 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting d2861f6f996f438cb7bbfae84ddd9e30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731544062240 2024-11-14T00:27:45,214 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b6c672bcc2241a89160d3d721801444, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731544062240 2024-11-14T00:27:45,215 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55659d32c36347f7a78fa3f359b3c70e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731544063486 2024-11-14T00:27:45,215 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c657a8ec907c418ba117a26864c5669c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731544063497 2024-11-14T00:27:45,231 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#A#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:45,232 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/a6c972cc3c5a4e3e979c0b959bdc9fa8 is 50, key is test_row_0/A:col10/1731544063497/Put/seqid=0 2024-11-14T00:27:45,236 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:45,238 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/e4aeb3a159e84b4eaa8733de81885f1b is 50, key is test_row_0/B:col10/1731544063497/Put/seqid=0 2024-11-14T00:27:45,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741891_1067 (size=12765) 2024-11-14T00:27:45,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741892_1068 (size=12765) 2024-11-14T00:27:45,270 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/a6c972cc3c5a4e3e979c0b959bdc9fa8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/a6c972cc3c5a4e3e979c0b959bdc9fa8 2024-11-14T00:27:45,273 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/e4aeb3a159e84b4eaa8733de81885f1b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/e4aeb3a159e84b4eaa8733de81885f1b 2024-11-14T00:27:45,285 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/B of 285207bf1146874945959d9159bd86b2 into e4aeb3a159e84b4eaa8733de81885f1b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:45,285 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:45,285 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/B, priority=13, startTime=1731544065205; duration=0sec 2024-11-14T00:27:45,285 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:27:45,285 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:B 2024-11-14T00:27:45,285 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:45,285 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/A of 285207bf1146874945959d9159bd86b2 into a6c972cc3c5a4e3e979c0b959bdc9fa8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:45,286 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:45,286 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/A, priority=13, startTime=1731544065205; duration=0sec 2024-11-14T00:27:45,286 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:45,286 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:A 2024-11-14T00:27:45,286 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:45,287 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/C is initiating minor compaction (all files) 2024-11-14T00:27:45,287 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/C in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:45,287 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/35997686d672440f94e8fa01c8ce3378, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/a3c1275f29284116b91e94b8695667ae, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b10dbfa6055a48b18c77a99f10d34ca8] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=36.1 K 2024-11-14T00:27:45,287 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 35997686d672440f94e8fa01c8ce3378, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731544061014 2024-11-14T00:27:45,288 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting a3c1275f29284116b91e94b8695667ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731544062240 2024-11-14T00:27:45,288 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting b10dbfa6055a48b18c77a99f10d34ca8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731544063497 2024-11-14T00:27:45,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T00:27:45,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:45,304 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-14T00:27:45,305 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#C#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:45,306 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/0aef655f0c8940e6a1ee5296d815cd6a is 50, key is test_row_0/C:col10/1731544063497/Put/seqid=0 2024-11-14T00:27:45,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/b3ba008ae6db4a288c911adbed992dd8 is 50, key is test_row_0/A:col10/1731544064202/Put/seqid=0 2024-11-14T00:27:45,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741893_1069 (size=12765) 2024-11-14T00:27:45,334 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/0aef655f0c8940e6a1ee5296d815cd6a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/0aef655f0c8940e6a1ee5296d815cd6a 2024-11-14T00:27:45,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741894_1070 (size=12301) 2024-11-14T00:27:45,359 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/b3ba008ae6db4a288c911adbed992dd8 2024-11-14T00:27:45,360 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/C of 285207bf1146874945959d9159bd86b2 into 0aef655f0c8940e6a1ee5296d815cd6a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:45,360 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:45,360 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/C, priority=13, startTime=1731544065206; duration=0sec 2024-11-14T00:27:45,360 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:45,360 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:C 2024-11-14T00:27:45,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/9a6db4cd9d964441bfa546f38cf184b9 is 50, key is test_row_0/B:col10/1731544064202/Put/seqid=0 2024-11-14T00:27:45,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:45,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:45,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741895_1071 (size=12301) 2024-11-14T00:27:45,408 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/9a6db4cd9d964441bfa546f38cf184b9 2024-11-14T00:27:45,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/b443b5f146de4f39b2e46af5f89570a7 is 50, key is test_row_0/C:col10/1731544064202/Put/seqid=0 2024-11-14T00:27:45,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544125440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:45,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544125443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:45,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544125444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:45,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544125444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:45,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741896_1072 (size=12301) 2024-11-14T00:27:45,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544125555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:45,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544125556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:45,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544125556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544125556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:45,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T00:27:45,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544125766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544125766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544125766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:45,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544125775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:45,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:45,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:45,874 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/b443b5f146de4f39b2e46af5f89570a7 2024-11-14T00:27:45,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/b3ba008ae6db4a288c911adbed992dd8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/b3ba008ae6db4a288c911adbed992dd8 2024-11-14T00:27:45,897 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/b3ba008ae6db4a288c911adbed992dd8, entries=150, sequenceid=275, filesize=12.0 K 2024-11-14T00:27:45,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/9a6db4cd9d964441bfa546f38cf184b9 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9a6db4cd9d964441bfa546f38cf184b9 2024-11-14T00:27:45,907 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9a6db4cd9d964441bfa546f38cf184b9, entries=150, sequenceid=275, filesize=12.0 K 2024-11-14T00:27:45,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/b443b5f146de4f39b2e46af5f89570a7 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b443b5f146de4f39b2e46af5f89570a7 2024-11-14T00:27:45,916 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b443b5f146de4f39b2e46af5f89570a7, entries=150, sequenceid=275, filesize=12.0 K 2024-11-14T00:27:45,917 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=120.76 KB/123660 for 285207bf1146874945959d9159bd86b2 in 613ms, sequenceid=275, compaction requested=false 2024-11-14T00:27:45,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:45,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:45,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-14T00:27:45,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-14T00:27:45,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-14T00:27:45,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3960 sec 2024-11-14T00:27:45,926 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.4130 sec 2024-11-14T00:27:46,094 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:27:46,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T00:27:46,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:46,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/c8894f4c06774e9ab4d4cdd338e10c7f is 50, key is test_row_0/A:col10/1731544066092/Put/seqid=0 2024-11-14T00:27:46,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741897_1073 (size=14741) 2024-11-14T00:27:46,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544126129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544126175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544126175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544126175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544126285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544126285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544126285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544126285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544126495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544126496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544126496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544126496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/c8894f4c06774e9ab4d4cdd338e10c7f 2024-11-14T00:27:46,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/3f9408bcd1aa4760ab8cfd156855c258 is 50, key is test_row_0/B:col10/1731544066092/Put/seqid=0 2024-11-14T00:27:46,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741898_1074 (size=12301) 2024-11-14T00:27:46,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/3f9408bcd1aa4760ab8cfd156855c258 2024-11-14T00:27:46,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/e38a516d8cc74f1cb13624393fc019f0 is 50, key is test_row_0/C:col10/1731544066092/Put/seqid=0 2024-11-14T00:27:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T00:27:46,666 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:27:46,668 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:27:46,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741899_1075 (size=12301) 2024-11-14T00:27:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-14T00:27:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T00:27:46,676 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:27:46,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/e38a516d8cc74f1cb13624393fc019f0 2024-11-14T00:27:46,684 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:27:46,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:27:46,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/c8894f4c06774e9ab4d4cdd338e10c7f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/c8894f4c06774e9ab4d4cdd338e10c7f 2024-11-14T00:27:46,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/c8894f4c06774e9ab4d4cdd338e10c7f, entries=200, sequenceid=299, filesize=14.4 K 2024-11-14T00:27:46,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/3f9408bcd1aa4760ab8cfd156855c258 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/3f9408bcd1aa4760ab8cfd156855c258 2024-11-14T00:27:46,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/3f9408bcd1aa4760ab8cfd156855c258, entries=150, sequenceid=299, filesize=12.0 K 2024-11-14T00:27:46,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/e38a516d8cc74f1cb13624393fc019f0 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/e38a516d8cc74f1cb13624393fc019f0 2024-11-14T00:27:46,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/e38a516d8cc74f1cb13624393fc019f0, entries=150, sequenceid=299, filesize=12.0 K 2024-11-14T00:27:46,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=80.51 KB/82440 for 285207bf1146874945959d9159bd86b2 in 642ms, sequenceid=299, compaction requested=true 2024-11-14T00:27:46,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:46,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:27:46,737 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:46,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:46,737 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:46,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:27:46,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:46,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:27:46,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:27:46,738 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:46,738 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39807 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:46,738 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/A is initiating minor compaction (all files) 2024-11-14T00:27:46,738 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/B is initiating minor compaction (all files) 2024-11-14T00:27:46,738 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/A in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:46,738 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/B in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:46,738 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/a6c972cc3c5a4e3e979c0b959bdc9fa8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/b3ba008ae6db4a288c911adbed992dd8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/c8894f4c06774e9ab4d4cdd338e10c7f] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=38.9 K 2024-11-14T00:27:46,738 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/e4aeb3a159e84b4eaa8733de81885f1b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9a6db4cd9d964441bfa546f38cf184b9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/3f9408bcd1aa4760ab8cfd156855c258] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=36.5 K 2024-11-14T00:27:46,739 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6c972cc3c5a4e3e979c0b959bdc9fa8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731544063497 2024-11-14T00:27:46,739 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting e4aeb3a159e84b4eaa8733de81885f1b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731544063497 2024-11-14T00:27:46,739 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a6db4cd9d964441bfa546f38cf184b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731544064192 2024-11-14T00:27:46,739 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3ba008ae6db4a288c911adbed992dd8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731544064192 2024-11-14T00:27:46,740 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f9408bcd1aa4760ab8cfd156855c258, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731544065438 2024-11-14T00:27:46,740 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8894f4c06774e9ab4d4cdd338e10c7f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731544065438 2024-11-14T00:27:46,763 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:46,764 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/f66153e7b7c045bda1bfcc329c0e7325 is 50, key is test_row_0/B:col10/1731544066092/Put/seqid=0 2024-11-14T00:27:46,766 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#A#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:46,767 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/51c9a35c138d434ebecda43b09393190 is 50, key is test_row_0/A:col10/1731544066092/Put/seqid=0 2024-11-14T00:27:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T00:27:46,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741900_1076 (size=13017) 2024-11-14T00:27:46,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741901_1077 (size=13017) 2024-11-14T00:27:46,804 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/51c9a35c138d434ebecda43b09393190 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/51c9a35c138d434ebecda43b09393190 2024-11-14T00:27:46,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:46,807 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:27:46,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-14T00:27:46,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/474808b7daf0426fbe3a3f259aaa1a2f is 50, key is test_row_0/A:col10/1731544066806/Put/seqid=0 2024-11-14T00:27:46,819 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/A of 285207bf1146874945959d9159bd86b2 into 51c9a35c138d434ebecda43b09393190(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:46,819 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:46,820 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/A, priority=13, startTime=1731544066736; duration=0sec 2024-11-14T00:27:46,820 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:27:46,820 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:A 2024-11-14T00:27:46,820 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:27:46,826 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:27:46,826 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/C is initiating minor compaction (all files) 2024-11-14T00:27:46,826 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/C in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:46,826 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/0aef655f0c8940e6a1ee5296d815cd6a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b443b5f146de4f39b2e46af5f89570a7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/e38a516d8cc74f1cb13624393fc019f0] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=36.5 K 2024-11-14T00:27:46,827 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aef655f0c8940e6a1ee5296d815cd6a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731544063497 2024-11-14T00:27:46,827 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting b443b5f146de4f39b2e46af5f89570a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731544064192 2024-11-14T00:27:46,828 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting e38a516d8cc74f1cb13624393fc019f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731544065438 2024-11-14T00:27:46,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T00:27:46,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:46,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:46,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:46,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:46,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:46,850 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#C#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:46,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544126845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,852 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/657e1d70917a4919a4399f02fec90b78 is 50, key is test_row_0/C:col10/1731544066092/Put/seqid=0 2024-11-14T00:27:46,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544126850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544126852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544126852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741902_1078 (size=14741) 2024-11-14T00:27:46,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/474808b7daf0426fbe3a3f259aaa1a2f 2024-11-14T00:27:46,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741903_1079 (size=13017) 2024-11-14T00:27:46,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/66d1aefdaa8a4f49b730d47d9a957675 is 50, key is test_row_0/B:col10/1731544066806/Put/seqid=0 2024-11-14T00:27:46,880 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/657e1d70917a4919a4399f02fec90b78 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/657e1d70917a4919a4399f02fec90b78 2024-11-14T00:27:46,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741904_1080 (size=12301) 2024-11-14T00:27:46,887 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/C of 285207bf1146874945959d9159bd86b2 into 657e1d70917a4919a4399f02fec90b78(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:46,887 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:46,887 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/C, priority=13, startTime=1731544066737; duration=0sec 2024-11-14T00:27:46,887 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:46,888 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:C 2024-11-14T00:27:46,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544126955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544126965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544126965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544126965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:46,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:46,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T00:27:46,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:46,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:46,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:46,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:46,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:46,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T00:27:46,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:47,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T00:27:47,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:47,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:47,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:47,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:47,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:47,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544127166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:47,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544127176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:47,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544127176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:47,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:47,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544127176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:47,202 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/f66153e7b7c045bda1bfcc329c0e7325 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/f66153e7b7c045bda1bfcc329c0e7325 2024-11-14T00:27:47,211 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 285207bf1146874945959d9159bd86b2/B of 285207bf1146874945959d9159bd86b2 into f66153e7b7c045bda1bfcc329c0e7325(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:47,211 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:47,211 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/B, priority=13, startTime=1731544066737; duration=0sec 2024-11-14T00:27:47,211 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:47,211 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:B 2024-11-14T00:27:47,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/66d1aefdaa8a4f49b730d47d9a957675 2024-11-14T00:27:47,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/fec55fe12d334a1589cda00e00374b8c is 50, key is test_row_0/C:col10/1731544066806/Put/seqid=0 2024-11-14T00:27:47,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T00:27:47,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:47,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:47,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:47,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:47,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:47,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T00:27:47,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741905_1081 (size=12301) 2024-11-14T00:27:47,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/fec55fe12d334a1589cda00e00374b8c 2024-11-14T00:27:47,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/474808b7daf0426fbe3a3f259aaa1a2f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/474808b7daf0426fbe3a3f259aaa1a2f 2024-11-14T00:27:47,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/474808b7daf0426fbe3a3f259aaa1a2f, entries=200, sequenceid=315, filesize=14.4 K 2024-11-14T00:27:47,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/66d1aefdaa8a4f49b730d47d9a957675 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/66d1aefdaa8a4f49b730d47d9a957675 2024-11-14T00:27:47,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/66d1aefdaa8a4f49b730d47d9a957675, entries=150, sequenceid=315, filesize=12.0 K 2024-11-14T00:27:47,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/fec55fe12d334a1589cda00e00374b8c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/fec55fe12d334a1589cda00e00374b8c 2024-11-14T00:27:47,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/fec55fe12d334a1589cda00e00374b8c, entries=150, sequenceid=315, filesize=12.0 K 2024-11-14T00:27:47,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 285207bf1146874945959d9159bd86b2 in 558ms, sequenceid=315, compaction requested=false 2024-11-14T00:27:47,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:47,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T00:27:47,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:47,459 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T00:27:47,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/053720a9ce8c437ab5b93a16cf7239bb is 50, key is test_row_0/A:col10/1731544066841/Put/seqid=0 2024-11-14T00:27:47,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741906_1082 (size=12301) 2024-11-14T00:27:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:47,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:47,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544127493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:47,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544127498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:47,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544127498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:47,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544127500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:47,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544127605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:47,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:47,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544127616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:47,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544127616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:47,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544127616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,620 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T00:27:47,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544127815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:47,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:47,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544127825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544127825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:47,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:47,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544127825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:47,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:47,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:47,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:47,875 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/053720a9ce8c437ab5b93a16cf7239bb 2024-11-14T00:27:47,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/6010e42d6b2943eabeb6bc79f3085e7f is 50, key is test_row_0/B:col10/1731544066841/Put/seqid=0 2024-11-14T00:27:47,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741907_1083 (size=12301) 2024-11-14T00:27:48,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544128125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:48,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:48,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:48,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:48,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544128136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:48,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544128136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:48,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544128136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:48,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:48,319 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/6010e42d6b2943eabeb6bc79f3085e7f 2024-11-14T00:27:48,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/9ad516514c35410cb8462355080fdcc9 is 50, key is test_row_0/C:col10/1731544066841/Put/seqid=0 2024-11-14T00:27:48,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741908_1084 (size=12301) 2024-11-14T00:27:48,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:48,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544128635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:48,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:48,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544128648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:48,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544128648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:48,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:48,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:48,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:48,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544128650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:48,789 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/9ad516514c35410cb8462355080fdcc9 2024-11-14T00:27:48,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/053720a9ce8c437ab5b93a16cf7239bb as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/053720a9ce8c437ab5b93a16cf7239bb 2024-11-14T00:27:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T00:27:48,838 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/053720a9ce8c437ab5b93a16cf7239bb, entries=150, sequenceid=339, filesize=12.0 K 2024-11-14T00:27:48,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/6010e42d6b2943eabeb6bc79f3085e7f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/6010e42d6b2943eabeb6bc79f3085e7f 2024-11-14T00:27:48,857 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/6010e42d6b2943eabeb6bc79f3085e7f, entries=150, sequenceid=339, filesize=12.0 K 2024-11-14T00:27:48,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/9ad516514c35410cb8462355080fdcc9 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/9ad516514c35410cb8462355080fdcc9 2024-11-14T00:27:48,874 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/9ad516514c35410cb8462355080fdcc9, entries=150, sequenceid=339, filesize=12.0 K 2024-11-14T00:27:48,877 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 285207bf1146874945959d9159bd86b2 in 1418ms, sequenceid=339, compaction requested=true 2024-11-14T00:27:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-14T00:27:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-14T00:27:48,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-14T00:27:48,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2000 sec 2024-11-14T00:27:48,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.2210 sec 2024-11-14T00:27:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:49,664 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:27:49,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-14T00:27:49,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/dfe6a5432de54893857ba97fa1778f58 is 50, key is test_row_0/A:col10/1731544067495/Put/seqid=0 2024-11-14T00:27:49,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741909_1085 (size=12301) 2024-11-14T00:27:49,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/dfe6a5432de54893857ba97fa1778f58 2024-11-14T00:27:49,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544129726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:49,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544129727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:49,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544129727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:49,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:49,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:49,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:49,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544129736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:49,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/4437e3fd03b9435084791392bac8e4ad is 50, key is test_row_0/B:col10/1731544067495/Put/seqid=0 2024-11-14T00:27:49,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741910_1086 (size=12301) 2024-11-14T00:27:49,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544129847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:49,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544129847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:49,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:49,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:49,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:49,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544129850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:49,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544129851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:49,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:49,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:49,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:49,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:49,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:49,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:49,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:49,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:49,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:50,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544130060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:50,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544130067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:50,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544130067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:50,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544130068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:50,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/4437e3fd03b9435084791392bac8e4ad 2024-11-14T00:27:50,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/ae481068d8ce4d518a22857ceafc9956 is 50, key is test_row_0/C:col10/1731544067495/Put/seqid=0 2024-11-14T00:27:50,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741911_1087 (size=12301) 2024-11-14T00:27:50,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/ae481068d8ce4d518a22857ceafc9956 2024-11-14T00:27:50,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/dfe6a5432de54893857ba97fa1778f58 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/dfe6a5432de54893857ba97fa1778f58 2024-11-14T00:27:50,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/dfe6a5432de54893857ba97fa1778f58, entries=150, sequenceid=356, filesize=12.0 K 2024-11-14T00:27:50,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/4437e3fd03b9435084791392bac8e4ad as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/4437e3fd03b9435084791392bac8e4ad 2024-11-14T00:27:50,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:50,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544130377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:50,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:50,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544130377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:50,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:50,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:50,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544130377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:50,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:50,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544130385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:50,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/4437e3fd03b9435084791392bac8e4ad, entries=150, sequenceid=356, filesize=12.0 K 2024-11-14T00:27:50,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/ae481068d8ce4d518a22857ceafc9956 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ae481068d8ce4d518a22857ceafc9956 2024-11-14T00:27:50,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ae481068d8ce4d518a22857ceafc9956, entries=150, sequenceid=356, filesize=12.0 K 2024-11-14T00:27:50,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 285207bf1146874945959d9159bd86b2 in 797ms, sequenceid=356, compaction requested=true 2024-11-14T00:27:50,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:50,463 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:27:50,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:27:50,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:50,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:27:50,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:27:50,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 285207bf1146874945959d9159bd86b2:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:27:50,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:27:50,468 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:27:50,481 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:27:50,481 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/A is initiating minor compaction (all files) 2024-11-14T00:27:50,481 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/A in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:50,481 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/51c9a35c138d434ebecda43b09393190, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/474808b7daf0426fbe3a3f259aaa1a2f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/053720a9ce8c437ab5b93a16cf7239bb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/dfe6a5432de54893857ba97fa1778f58] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=51.1 K 2024-11-14T00:27:50,485 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:27:50,485 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/B is initiating minor compaction (all files) 2024-11-14T00:27:50,485 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/B in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:50,485 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/f66153e7b7c045bda1bfcc329c0e7325, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/66d1aefdaa8a4f49b730d47d9a957675, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/6010e42d6b2943eabeb6bc79f3085e7f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/4437e3fd03b9435084791392bac8e4ad] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=48.8 K 2024-11-14T00:27:50,486 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting f66153e7b7c045bda1bfcc329c0e7325, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731544065438 2024-11-14T00:27:50,486 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 51c9a35c138d434ebecda43b09393190, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731544065438 2024-11-14T00:27:50,491 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 474808b7daf0426fbe3a3f259aaa1a2f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731544066129 2024-11-14T00:27:50,492 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66d1aefdaa8a4f49b730d47d9a957675, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731544066129 2024-11-14T00:27:50,495 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6010e42d6b2943eabeb6bc79f3085e7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731544066841 2024-11-14T00:27:50,495 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 053720a9ce8c437ab5b93a16cf7239bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731544066841 2024-11-14T00:27:50,500 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting dfe6a5432de54893857ba97fa1778f58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731544067488 2024-11-14T00:27:50,500 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4437e3fd03b9435084791392bac8e4ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731544067488 2024-11-14T00:27:50,526 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:50,529 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/e21294576b0f4e2bb33731d95b409712 is 50, key is test_row_0/A:col10/1731544067495/Put/seqid=0 2024-11-14T00:27:50,530 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#B#compaction#76 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:50,531 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/98f8be452e2240b28ff6a4deed4dbdce is 50, key is test_row_0/B:col10/1731544067495/Put/seqid=0 2024-11-14T00:27:50,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741912_1088 (size=13153) 2024-11-14T00:27:50,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741913_1089 (size=13153) 2024-11-14T00:27:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T00:27:50,849 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:27:50,852 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:27:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-14T00:27:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T00:27:50,862 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:27:50,864 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:27:50,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:27:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:50,900 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:27:50,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-14T00:27:50,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/9ddcae3e5d7d4df8bec35bea90e69bf5 is 50, key is test_row_0/A:col10/1731544070891/Put/seqid=0 2024-11-14T00:27:50,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544130947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:50,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:50,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544130955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:50,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:50,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:50,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544130958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:50,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:50,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544130959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T00:27:50,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741914_1090 (size=14741) 2024-11-14T00:27:51,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T00:27:51,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:51,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:51,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:51,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,026 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/98f8be452e2240b28ff6a4deed4dbdce as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/98f8be452e2240b28ff6a4deed4dbdce 2024-11-14T00:27:51,037 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 285207bf1146874945959d9159bd86b2/B of 285207bf1146874945959d9159bd86b2 into 98f8be452e2240b28ff6a4deed4dbdce(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:51,037 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:51,037 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/B, priority=12, startTime=1731544070465; duration=0sec 2024-11-14T00:27:51,037 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:27:51,037 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:B 2024-11-14T00:27:51,037 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:27:51,042 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/e21294576b0f4e2bb33731d95b409712 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/e21294576b0f4e2bb33731d95b409712 2024-11-14T00:27:51,050 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 285207bf1146874945959d9159bd86b2/A of 285207bf1146874945959d9159bd86b2 into e21294576b0f4e2bb33731d95b409712(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:51,050 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:51,051 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/A, priority=12, startTime=1731544070462; duration=0sec 2024-11-14T00:27:51,051 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:51,051 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:A 2024-11-14T00:27:51,053 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:27:51,053 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 285207bf1146874945959d9159bd86b2/C is initiating minor compaction (all files) 2024-11-14T00:27:51,054 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 285207bf1146874945959d9159bd86b2/C in TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:51,054 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/657e1d70917a4919a4399f02fec90b78, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/fec55fe12d334a1589cda00e00374b8c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/9ad516514c35410cb8462355080fdcc9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ae481068d8ce4d518a22857ceafc9956] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp, totalSize=48.8 K 2024-11-14T00:27:51,054 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 657e1d70917a4919a4399f02fec90b78, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731544065438 2024-11-14T00:27:51,055 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting fec55fe12d334a1589cda00e00374b8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731544066129 2024-11-14T00:27:51,055 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ad516514c35410cb8462355080fdcc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731544066841 2024-11-14T00:27:51,056 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae481068d8ce4d518a22857ceafc9956, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731544067488 2024-11-14T00:27:51,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544131076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544131076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:51,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544131077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:51,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544131077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:51,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:51,087 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 285207bf1146874945959d9159bd86b2#C#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:27:51,088 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/b2504011c9594d648c7b0380216d608e is 50, key is test_row_0/C:col10/1731544067495/Put/seqid=0 2024-11-14T00:27:51,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741915_1091 (size=13153) 2024-11-14T00:27:51,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T00:27:51,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:51,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:51,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:51,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T00:27:51,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544131290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:51,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544131290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:51,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544131291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:51,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:51,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544131291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:51,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:51,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T00:27:51,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:51,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:51,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:51,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/9ddcae3e5d7d4df8bec35bea90e69bf5 2024-11-14T00:27:51,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/ecd70252a0334e31a09b44edfcf1ed5b is 50, key is test_row_0/B:col10/1731544070891/Put/seqid=0 2024-11-14T00:27:51,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741916_1092 (size=12301) 2024-11-14T00:27:51,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/ecd70252a0334e31a09b44edfcf1ed5b 2024-11-14T00:27:51,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T00:27:51,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T00:27:51,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:51,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:51,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/ac0296d33be44179a3f60ef46febb3f3 is 50, key is test_row_0/C:col10/1731544070891/Put/seqid=0 2024-11-14T00:27:51,546 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/b2504011c9594d648c7b0380216d608e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b2504011c9594d648c7b0380216d608e 2024-11-14T00:27:51,566 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 285207bf1146874945959d9159bd86b2/C of 285207bf1146874945959d9159bd86b2 into b2504011c9594d648c7b0380216d608e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:27:51,566 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:51,566 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., storeName=285207bf1146874945959d9159bd86b2/C, priority=12, startTime=1731544070465; duration=0sec 2024-11-14T00:27:51,566 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:27:51,567 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 285207bf1146874945959d9159bd86b2:C 2024-11-14T00:27:51,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741917_1093 (size=12301) 2024-11-14T00:27:51,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58726 deadline: 1731544131608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:51,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:51,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58786 deadline: 1731544131615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:51,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58770 deadline: 1731544131616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:51,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:27:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:58716 deadline: 1731544131619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:51,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:27:51,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:27:51,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T00:27:51,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:51,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:51,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:51,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T00:27:51,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:51,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:51,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:51,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,815 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-14T00:27:51,816 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:27:51,821 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-14T00:27:51,821 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-14T00:27:51,821 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:27:51,821 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:27:51,822 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-14T00:27:51,822 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:27:51,823 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:51,823 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:51,823 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:51,823 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:51,824 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:51,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:27:51,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:27:51,825 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:51,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:27:51,824 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:51,826 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:51,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:27:51,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T00:27:51,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:51,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:51,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:51,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:27:51,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/ac0296d33be44179a3f60ef46febb3f3 2024-11-14T00:27:51,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/9ddcae3e5d7d4df8bec35bea90e69bf5 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/9ddcae3e5d7d4df8bec35bea90e69bf5 2024-11-14T00:27:52,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T00:27:52,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/9ddcae3e5d7d4df8bec35bea90e69bf5, entries=200, sequenceid=376, filesize=14.4 K 2024-11-14T00:27:52,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/ecd70252a0334e31a09b44edfcf1ed5b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/ecd70252a0334e31a09b44edfcf1ed5b 2024-11-14T00:27:52,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/ecd70252a0334e31a09b44edfcf1ed5b, entries=150, sequenceid=376, filesize=12.0 K 2024-11-14T00:27:52,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/ac0296d33be44179a3f60ef46febb3f3 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ac0296d33be44179a3f60ef46febb3f3 2024-11-14T00:27:52,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ac0296d33be44179a3f60ef46febb3f3, entries=150, sequenceid=376, filesize=12.0 K 2024-11-14T00:27:52,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 285207bf1146874945959d9159bd86b2 in 1159ms, sequenceid=376, compaction requested=false 2024-11-14T00:27:52,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:52,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T00:27:52,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:52,115 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-14T00:27:52,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/f23da6fcd338455b9fed38946de3ece8 is 50, key is test_row_0/A:col10/1731544070953/Put/seqid=0 2024-11-14T00:27:52,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:52,129 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-14T00:27:52,129 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:27:52,129 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:52,129 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:52,129 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:27:52,131 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-14T00:27:52,131 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:27:52,131 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:52,131 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-14T00:27:52,131 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:52,131 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:27:52,131 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:27:52,131 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:52,132 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:52,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:27:52,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. as already flushing 2024-11-14T00:27:52,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741918_1094 (size=12301) 2024-11-14T00:27:52,134 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/f23da6fcd338455b9fed38946de3ece8 2024-11-14T00:27:52,138 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-14T00:27:52,138 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:27:52,138 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:52,138 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:52,138 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:27:52,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/f0736d704edd4f0989e1a765b978fc81 is 50, key is test_row_0/B:col10/1731544070953/Put/seqid=0 2024-11-14T00:27:52,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741919_1095 (size=12301) 2024-11-14T00:27:52,567 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/f0736d704edd4f0989e1a765b978fc81 2024-11-14T00:27:52,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/ccff9dc510ee4ad4a455e5aa12b5b762 is 50, key is test_row_0/C:col10/1731544070953/Put/seqid=0 2024-11-14T00:27:52,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741920_1096 (size=12301) 2024-11-14T00:27:52,992 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/ccff9dc510ee4ad4a455e5aa12b5b762 2024-11-14T00:27:53,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/f23da6fcd338455b9fed38946de3ece8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/f23da6fcd338455b9fed38946de3ece8 2024-11-14T00:27:53,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T00:27:53,024 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/f23da6fcd338455b9fed38946de3ece8, entries=150, sequenceid=396, filesize=12.0 K 2024-11-14T00:27:53,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/f0736d704edd4f0989e1a765b978fc81 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/f0736d704edd4f0989e1a765b978fc81 2024-11-14T00:27:53,056 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/f0736d704edd4f0989e1a765b978fc81, entries=150, sequenceid=396, filesize=12.0 K 2024-11-14T00:27:53,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/ccff9dc510ee4ad4a455e5aa12b5b762 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ccff9dc510ee4ad4a455e5aa12b5b762 2024-11-14T00:27:53,086 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ccff9dc510ee4ad4a455e5aa12b5b762, entries=150, sequenceid=396, filesize=12.0 K 2024-11-14T00:27:53,087 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=26.84 KB/27480 for 285207bf1146874945959d9159bd86b2 in 972ms, sequenceid=396, compaction requested=true 2024-11-14T00:27:53,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 285207bf1146874945959d9159bd86b2: 2024-11-14T00:27:53,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:53,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-14T00:27:53,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-14T00:27:53,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-14T00:27:53,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2310 sec 2024-11-14T00:27:53,109 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.2550 sec 2024-11-14T00:27:53,206 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T00:27:54,548 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-14T00:27:54,548 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:27:54,548 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:54,548 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:54,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:27:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T00:27:55,036 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:27:55,036 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T00:27:55,036 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-14T00:27:55,036 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-14T00:27:55,036 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-14T00:27:55,037 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-14T00:27:55,037 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-14T00:27:55,037 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T00:27:55,037 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1896 2024-11-14T00:27:55,037 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1951 2024-11-14T00:27:55,037 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T00:27:55,037 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 945 2024-11-14T00:27:55,037 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2834 rows 2024-11-14T00:27:55,037 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 961 2024-11-14T00:27:55,037 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2882 rows 2024-11-14T00:27:55,037 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T00:27:55,037 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T00:27:55,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:55,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:55,038 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:27:55,044 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-14T00:27:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T00:27:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T00:27:55,057 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544075056"}]},"ts":"1731544075056"} 2024-11-14T00:27:55,060 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T00:27:55,060 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T00:27:55,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T00:27:55,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=285207bf1146874945959d9159bd86b2, UNASSIGN}] 2024-11-14T00:27:55,072 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=285207bf1146874945959d9159bd86b2, UNASSIGN 2024-11-14T00:27:55,074 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=285207bf1146874945959d9159bd86b2, regionState=CLOSING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:55,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=285207bf1146874945959d9159bd86b2, UNASSIGN because future has completed 2024-11-14T00:27:55,083 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T00:27:55,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:27:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T00:27:55,250 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(122): Close 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:55,250 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T00:27:55,251 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1722): Closing 285207bf1146874945959d9159bd86b2, disabling compactions & flushes 2024-11-14T00:27:55,251 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:55,251 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:55,251 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. after waiting 0 ms 2024-11-14T00:27:55,251 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:55,251 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(2902): Flushing 285207bf1146874945959d9159bd86b2 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-14T00:27:55,263 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/e06823b0cbbf4e7684177364c7141898 is 50, key is test_row_0/A:col10/1731544072128/Put/seqid=0 2024-11-14T00:27:55,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741921_1097 (size=12301) 2024-11-14T00:27:55,296 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/e06823b0cbbf4e7684177364c7141898 2024-11-14T00:27:55,311 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/14ddc12d7f404a23b09bd98d789d0fed is 50, key is test_row_0/B:col10/1731544072128/Put/seqid=0 2024-11-14T00:27:55,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741922_1098 (size=12301) 2024-11-14T00:27:55,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T00:27:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T00:27:55,739 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/14ddc12d7f404a23b09bd98d789d0fed 2024-11-14T00:27:55,749 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/5b7478739eaf404c90f51d291112b024 is 50, key is test_row_0/C:col10/1731544072128/Put/seqid=0 2024-11-14T00:27:55,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741923_1099 (size=12301) 2024-11-14T00:27:55,762 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/5b7478739eaf404c90f51d291112b024 2024-11-14T00:27:55,768 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/A/e06823b0cbbf4e7684177364c7141898 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/e06823b0cbbf4e7684177364c7141898 2024-11-14T00:27:55,774 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/e06823b0cbbf4e7684177364c7141898, entries=150, sequenceid=404, filesize=12.0 K 2024-11-14T00:27:55,776 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/B/14ddc12d7f404a23b09bd98d789d0fed as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/14ddc12d7f404a23b09bd98d789d0fed 2024-11-14T00:27:55,782 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/14ddc12d7f404a23b09bd98d789d0fed, entries=150, sequenceid=404, filesize=12.0 K 2024-11-14T00:27:55,785 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/.tmp/C/5b7478739eaf404c90f51d291112b024 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/5b7478739eaf404c90f51d291112b024 2024-11-14T00:27:55,792 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/5b7478739eaf404c90f51d291112b024, entries=150, sequenceid=404, filesize=12.0 K 2024-11-14T00:27:55,793 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 285207bf1146874945959d9159bd86b2 in 542ms, sequenceid=404, compaction requested=true 2024-11-14T00:27:55,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/790dc60e3a3d43aead8f442ee25d4abd, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/a1f11c478bc74689854e90b265d2357a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/92929719fe654569bd5e09d0ada85b16, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/7daa78d615c24969a29c6a9a16e61592, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/491df79c0ede4643a7ea0b525da8119e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0ced9a1a4db74d968dda0f7045450492, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/086a919a11304199beec073084346157, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0d8c4d88034f4d438dabc9e100af1a9b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/8ac78c1b0e524160840c2dae49ddf7a2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0e7940092e2c4c41adec1d1aa1c0012c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/5320e84f839d4af8b41ec4048ab53962, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/2bd9e3f3ea6c4f6ebd0432cbb1486fad, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/1740c2734157494a99b3a4f397c02209, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/3856a4c6994141e9a9f21ec39f7682f5, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/8570d31b51634923839a37b3239d9970, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/237df47a958f4c3d82af6abd25f06bfe, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/7b6c672bcc2241a89160d3d721801444, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/55659d32c36347f7a78fa3f359b3c70e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/a6c972cc3c5a4e3e979c0b959bdc9fa8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/b3ba008ae6db4a288c911adbed992dd8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/c8894f4c06774e9ab4d4cdd338e10c7f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/51c9a35c138d434ebecda43b09393190, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/474808b7daf0426fbe3a3f259aaa1a2f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/053720a9ce8c437ab5b93a16cf7239bb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/dfe6a5432de54893857ba97fa1778f58] to archive 2024-11-14T00:27:55,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:27:55,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/790dc60e3a3d43aead8f442ee25d4abd to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/790dc60e3a3d43aead8f442ee25d4abd 2024-11-14T00:27:55,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/a1f11c478bc74689854e90b265d2357a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/a1f11c478bc74689854e90b265d2357a 2024-11-14T00:27:55,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/92929719fe654569bd5e09d0ada85b16 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/92929719fe654569bd5e09d0ada85b16 2024-11-14T00:27:55,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/7daa78d615c24969a29c6a9a16e61592 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/7daa78d615c24969a29c6a9a16e61592 2024-11-14T00:27:55,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/491df79c0ede4643a7ea0b525da8119e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/491df79c0ede4643a7ea0b525da8119e 2024-11-14T00:27:55,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0ced9a1a4db74d968dda0f7045450492 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0ced9a1a4db74d968dda0f7045450492 2024-11-14T00:27:55,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/086a919a11304199beec073084346157 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/086a919a11304199beec073084346157 2024-11-14T00:27:55,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0d8c4d88034f4d438dabc9e100af1a9b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0d8c4d88034f4d438dabc9e100af1a9b 2024-11-14T00:27:55,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/8ac78c1b0e524160840c2dae49ddf7a2 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/8ac78c1b0e524160840c2dae49ddf7a2 2024-11-14T00:27:55,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0e7940092e2c4c41adec1d1aa1c0012c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/0e7940092e2c4c41adec1d1aa1c0012c 2024-11-14T00:27:55,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/5320e84f839d4af8b41ec4048ab53962 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/5320e84f839d4af8b41ec4048ab53962 2024-11-14T00:27:55,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/2bd9e3f3ea6c4f6ebd0432cbb1486fad to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/2bd9e3f3ea6c4f6ebd0432cbb1486fad 2024-11-14T00:27:55,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/1740c2734157494a99b3a4f397c02209 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/1740c2734157494a99b3a4f397c02209 2024-11-14T00:27:55,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/3856a4c6994141e9a9f21ec39f7682f5 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/3856a4c6994141e9a9f21ec39f7682f5 2024-11-14T00:27:55,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/8570d31b51634923839a37b3239d9970 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/8570d31b51634923839a37b3239d9970 2024-11-14T00:27:55,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/237df47a958f4c3d82af6abd25f06bfe to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/237df47a958f4c3d82af6abd25f06bfe 2024-11-14T00:27:55,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/7b6c672bcc2241a89160d3d721801444 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/7b6c672bcc2241a89160d3d721801444 2024-11-14T00:27:55,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/55659d32c36347f7a78fa3f359b3c70e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/55659d32c36347f7a78fa3f359b3c70e 2024-11-14T00:27:55,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/a6c972cc3c5a4e3e979c0b959bdc9fa8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/a6c972cc3c5a4e3e979c0b959bdc9fa8 2024-11-14T00:27:55,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/b3ba008ae6db4a288c911adbed992dd8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/b3ba008ae6db4a288c911adbed992dd8 2024-11-14T00:27:55,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/c8894f4c06774e9ab4d4cdd338e10c7f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/c8894f4c06774e9ab4d4cdd338e10c7f 2024-11-14T00:27:55,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/51c9a35c138d434ebecda43b09393190 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/51c9a35c138d434ebecda43b09393190 2024-11-14T00:27:55,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/474808b7daf0426fbe3a3f259aaa1a2f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/474808b7daf0426fbe3a3f259aaa1a2f 2024-11-14T00:27:55,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/053720a9ce8c437ab5b93a16cf7239bb to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/053720a9ce8c437ab5b93a16cf7239bb 2024-11-14T00:27:55,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/dfe6a5432de54893857ba97fa1778f58 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/dfe6a5432de54893857ba97fa1778f58 2024-11-14T00:27:55,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9e2a998618e24966abcb1665ee095191, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9238b55a2969427ab3ff4b279461ca96, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/80427010fc5044f199540ed10bc3c7a2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/cfb756f62c2549e0ac4ee7c534aade10, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/989f879c0758472da6ff5a260d34ff86, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9c90db9c64a144f9a46b44bea09a5fff, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/f51889d9bc4046f79df5d5cf24aa3c22, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/970cd479196e4c48be8a09704e6b97bb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/1e8e8196a3544aa09b4f3ac96df52e36, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/b5a92b6858914b428a25ff2275e74c5c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/cfb2c42fe91e4e399be52c252b80a597, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/76acba3d34824f6ebddfccfe01308bd8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/c4c8ea6bf3ca43bb8b2bd191292c08a6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/5359ee3545d649979f8e19c86adc9dd8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/79983163999047769379c6a67fba9eec, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/2b58d3515b514ae6b7a4bcc7f6180d65, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/d2861f6f996f438cb7bbfae84ddd9e30, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/e4aeb3a159e84b4eaa8733de81885f1b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/c657a8ec907c418ba117a26864c5669c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9a6db4cd9d964441bfa546f38cf184b9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/f66153e7b7c045bda1bfcc329c0e7325, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/3f9408bcd1aa4760ab8cfd156855c258, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/66d1aefdaa8a4f49b730d47d9a957675, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/6010e42d6b2943eabeb6bc79f3085e7f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/4437e3fd03b9435084791392bac8e4ad] to archive 2024-11-14T00:27:55,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:27:55,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9e2a998618e24966abcb1665ee095191 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9e2a998618e24966abcb1665ee095191 2024-11-14T00:27:55,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9238b55a2969427ab3ff4b279461ca96 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9238b55a2969427ab3ff4b279461ca96 2024-11-14T00:27:55,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/80427010fc5044f199540ed10bc3c7a2 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/80427010fc5044f199540ed10bc3c7a2 2024-11-14T00:27:55,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/cfb756f62c2549e0ac4ee7c534aade10 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/cfb756f62c2549e0ac4ee7c534aade10 2024-11-14T00:27:55,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/989f879c0758472da6ff5a260d34ff86 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/989f879c0758472da6ff5a260d34ff86 2024-11-14T00:27:55,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9c90db9c64a144f9a46b44bea09a5fff to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9c90db9c64a144f9a46b44bea09a5fff 2024-11-14T00:27:55,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/f51889d9bc4046f79df5d5cf24aa3c22 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/f51889d9bc4046f79df5d5cf24aa3c22 2024-11-14T00:27:55,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/970cd479196e4c48be8a09704e6b97bb to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/970cd479196e4c48be8a09704e6b97bb 2024-11-14T00:27:55,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/1e8e8196a3544aa09b4f3ac96df52e36 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/1e8e8196a3544aa09b4f3ac96df52e36 2024-11-14T00:27:55,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/b5a92b6858914b428a25ff2275e74c5c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/b5a92b6858914b428a25ff2275e74c5c 2024-11-14T00:27:55,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/cfb2c42fe91e4e399be52c252b80a597 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/cfb2c42fe91e4e399be52c252b80a597 2024-11-14T00:27:55,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/76acba3d34824f6ebddfccfe01308bd8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/76acba3d34824f6ebddfccfe01308bd8 2024-11-14T00:27:55,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/c4c8ea6bf3ca43bb8b2bd191292c08a6 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/c4c8ea6bf3ca43bb8b2bd191292c08a6 2024-11-14T00:27:55,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/5359ee3545d649979f8e19c86adc9dd8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/5359ee3545d649979f8e19c86adc9dd8 2024-11-14T00:27:55,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/79983163999047769379c6a67fba9eec to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/79983163999047769379c6a67fba9eec 2024-11-14T00:27:55,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/2b58d3515b514ae6b7a4bcc7f6180d65 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/2b58d3515b514ae6b7a4bcc7f6180d65 2024-11-14T00:27:55,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/d2861f6f996f438cb7bbfae84ddd9e30 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/d2861f6f996f438cb7bbfae84ddd9e30 2024-11-14T00:27:55,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/e4aeb3a159e84b4eaa8733de81885f1b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/e4aeb3a159e84b4eaa8733de81885f1b 2024-11-14T00:27:55,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/c657a8ec907c418ba117a26864c5669c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/c657a8ec907c418ba117a26864c5669c 2024-11-14T00:27:55,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9a6db4cd9d964441bfa546f38cf184b9 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/9a6db4cd9d964441bfa546f38cf184b9 2024-11-14T00:27:55,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/f66153e7b7c045bda1bfcc329c0e7325 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/f66153e7b7c045bda1bfcc329c0e7325 2024-11-14T00:27:55,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/3f9408bcd1aa4760ab8cfd156855c258 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/3f9408bcd1aa4760ab8cfd156855c258 2024-11-14T00:27:55,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/66d1aefdaa8a4f49b730d47d9a957675 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/66d1aefdaa8a4f49b730d47d9a957675 2024-11-14T00:27:55,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/6010e42d6b2943eabeb6bc79f3085e7f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/6010e42d6b2943eabeb6bc79f3085e7f 2024-11-14T00:27:56,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/4437e3fd03b9435084791392bac8e4ad to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/4437e3fd03b9435084791392bac8e4ad 2024-11-14T00:27:56,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/1d71d3cca089429697d99cffd0e5115a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/167b1bb160b943bdba2a6bd82ea01718, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/06265e744d6146d796302c01a4d222a0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/394178c30f5940cc97822b7919c58607, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ebcf000e638d4f9c9b8a52f9b84e633a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/3026c3739a9045a89fcc1058398dff75, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/5160e939330043109eac4901eb7ef3e9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/35d72ce9416644bead9b38f0dafd044d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/a1480ac9a9ef47e4bb10841e471aa854, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/f9a7d44607184ddabddcd1b6d251aee5, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b7a6d015598f45bca00e81fbcce5355e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/fb15d6f96893448d976998b1a8023543, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b9c3048a8f4545e4be97a8bd239e0670, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/066fc4dbd099496cbb4d6c51f6052ae9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/35997686d672440f94e8fa01c8ce3378, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/6b85b0806ed34988af7faad5c94d88e8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/a3c1275f29284116b91e94b8695667ae, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/0aef655f0c8940e6a1ee5296d815cd6a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b10dbfa6055a48b18c77a99f10d34ca8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b443b5f146de4f39b2e46af5f89570a7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/657e1d70917a4919a4399f02fec90b78, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/e38a516d8cc74f1cb13624393fc019f0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/fec55fe12d334a1589cda00e00374b8c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/9ad516514c35410cb8462355080fdcc9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ae481068d8ce4d518a22857ceafc9956] to archive 2024-11-14T00:27:56,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:27:56,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/1d71d3cca089429697d99cffd0e5115a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/1d71d3cca089429697d99cffd0e5115a 2024-11-14T00:27:56,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/167b1bb160b943bdba2a6bd82ea01718 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/167b1bb160b943bdba2a6bd82ea01718 2024-11-14T00:27:56,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/06265e744d6146d796302c01a4d222a0 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/06265e744d6146d796302c01a4d222a0 2024-11-14T00:27:56,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/394178c30f5940cc97822b7919c58607 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/394178c30f5940cc97822b7919c58607 2024-11-14T00:27:56,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ebcf000e638d4f9c9b8a52f9b84e633a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ebcf000e638d4f9c9b8a52f9b84e633a 2024-11-14T00:27:56,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/3026c3739a9045a89fcc1058398dff75 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/3026c3739a9045a89fcc1058398dff75 2024-11-14T00:27:56,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/5160e939330043109eac4901eb7ef3e9 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/5160e939330043109eac4901eb7ef3e9 2024-11-14T00:27:56,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/35d72ce9416644bead9b38f0dafd044d to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/35d72ce9416644bead9b38f0dafd044d 2024-11-14T00:27:56,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/a1480ac9a9ef47e4bb10841e471aa854 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/a1480ac9a9ef47e4bb10841e471aa854 2024-11-14T00:27:56,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/f9a7d44607184ddabddcd1b6d251aee5 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/f9a7d44607184ddabddcd1b6d251aee5 2024-11-14T00:27:56,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b7a6d015598f45bca00e81fbcce5355e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b7a6d015598f45bca00e81fbcce5355e 2024-11-14T00:27:56,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/fb15d6f96893448d976998b1a8023543 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/fb15d6f96893448d976998b1a8023543 2024-11-14T00:27:56,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b9c3048a8f4545e4be97a8bd239e0670 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b9c3048a8f4545e4be97a8bd239e0670 2024-11-14T00:27:56,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/066fc4dbd099496cbb4d6c51f6052ae9 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/066fc4dbd099496cbb4d6c51f6052ae9 2024-11-14T00:27:56,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/35997686d672440f94e8fa01c8ce3378 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/35997686d672440f94e8fa01c8ce3378 2024-11-14T00:27:56,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/6b85b0806ed34988af7faad5c94d88e8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/6b85b0806ed34988af7faad5c94d88e8 2024-11-14T00:27:56,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/a3c1275f29284116b91e94b8695667ae to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/a3c1275f29284116b91e94b8695667ae 2024-11-14T00:27:56,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/0aef655f0c8940e6a1ee5296d815cd6a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/0aef655f0c8940e6a1ee5296d815cd6a 2024-11-14T00:27:56,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b10dbfa6055a48b18c77a99f10d34ca8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b10dbfa6055a48b18c77a99f10d34ca8 2024-11-14T00:27:56,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b443b5f146de4f39b2e46af5f89570a7 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b443b5f146de4f39b2e46af5f89570a7 2024-11-14T00:27:56,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/657e1d70917a4919a4399f02fec90b78 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/657e1d70917a4919a4399f02fec90b78 2024-11-14T00:27:56,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/e38a516d8cc74f1cb13624393fc019f0 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/e38a516d8cc74f1cb13624393fc019f0 2024-11-14T00:27:56,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/fec55fe12d334a1589cda00e00374b8c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/fec55fe12d334a1589cda00e00374b8c 2024-11-14T00:27:56,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/9ad516514c35410cb8462355080fdcc9 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/9ad516514c35410cb8462355080fdcc9 2024-11-14T00:27:56,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ae481068d8ce4d518a22857ceafc9956 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ae481068d8ce4d518a22857ceafc9956 2024-11-14T00:27:56,166 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/recovered.edits/407.seqid, newMaxSeqId=407, maxSeqId=1 2024-11-14T00:27:56,173 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. 2024-11-14T00:27:56,173 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1676): Region close journal for 285207bf1146874945959d9159bd86b2: Waiting for close lock at 1731544075251Running coprocessor pre-close hooks at 1731544075251Disabling compacts and flushes for region at 1731544075251Disabling writes for close at 1731544075251Obtaining lock to block concurrent updates at 1731544075251Preparing flush snapshotting stores in 285207bf1146874945959d9159bd86b2 at 1731544075251Finished memstore snapshotting TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731544075252 (+1 ms)Flushing stores of TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2. at 1731544075252Flushing 285207bf1146874945959d9159bd86b2/A: creating writer at 1731544075252Flushing 285207bf1146874945959d9159bd86b2/A: appending metadata at 1731544075262 (+10 ms)Flushing 285207bf1146874945959d9159bd86b2/A: closing flushed file at 1731544075262Flushing 285207bf1146874945959d9159bd86b2/B: creating writer at 1731544075304 (+42 ms)Flushing 285207bf1146874945959d9159bd86b2/B: appending metadata at 1731544075310 (+6 ms)Flushing 285207bf1146874945959d9159bd86b2/B: closing flushed file at 1731544075310Flushing 285207bf1146874945959d9159bd86b2/C: creating writer at 1731544075745 (+435 ms)Flushing 285207bf1146874945959d9159bd86b2/C: appending metadata at 1731544075748 (+3 ms)Flushing 285207bf1146874945959d9159bd86b2/C: closing flushed file at 1731544075748Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e38cdcb: reopening flushed file at 1731544075767 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@558f410: reopening flushed file at 1731544075775 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38bd567d: reopening flushed file at 1731544075782 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 285207bf1146874945959d9159bd86b2 in 542ms, sequenceid=404, compaction requested=true at 1731544075793 (+11 ms)Writing region close event to WAL at 1731544076145 (+352 ms)Running coprocessor post-close hooks at 1731544076173 (+28 ms)Closed at 1731544076173 2024-11-14T00:27:56,190 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(157): Closed 285207bf1146874945959d9159bd86b2 2024-11-14T00:27:56,190 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=285207bf1146874945959d9159bd86b2, regionState=CLOSED 2024-11-14T00:27:56,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:27:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T00:27:56,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-14T00:27:56,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseRegionProcedure 285207bf1146874945959d9159bd86b2, server=63baa11b75f3,33199,1731544045387 in 1.1140 sec 2024-11-14T00:27:56,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-14T00:27:56,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=285207bf1146874945959d9159bd86b2, UNASSIGN in 1.1310 sec 2024-11-14T00:27:56,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-14T00:27:56,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1450 sec 2024-11-14T00:27:56,215 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544076215"}]},"ts":"1731544076215"} 2024-11-14T00:27:56,221 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T00:27:56,222 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T00:27:56,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-14T00:27:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T00:27:57,205 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:27:57,209 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-14T00:27:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:27:57,216 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:27:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T00:27:57,218 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=25, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:27:57,226 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2 2024-11-14T00:27:57,276 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/recovered.edits] 2024-11-14T00:27:57,309 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/9ddcae3e5d7d4df8bec35bea90e69bf5 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/9ddcae3e5d7d4df8bec35bea90e69bf5 2024-11-14T00:27:57,323 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/e06823b0cbbf4e7684177364c7141898 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/e06823b0cbbf4e7684177364c7141898 2024-11-14T00:27:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T00:27:57,335 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/e21294576b0f4e2bb33731d95b409712 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/e21294576b0f4e2bb33731d95b409712 2024-11-14T00:27:57,338 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/f23da6fcd338455b9fed38946de3ece8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/A/f23da6fcd338455b9fed38946de3ece8 2024-11-14T00:27:57,342 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/14ddc12d7f404a23b09bd98d789d0fed to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/14ddc12d7f404a23b09bd98d789d0fed 2024-11-14T00:27:57,349 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/98f8be452e2240b28ff6a4deed4dbdce to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/98f8be452e2240b28ff6a4deed4dbdce 2024-11-14T00:27:57,350 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/ecd70252a0334e31a09b44edfcf1ed5b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/ecd70252a0334e31a09b44edfcf1ed5b 2024-11-14T00:27:57,353 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/f0736d704edd4f0989e1a765b978fc81 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/B/f0736d704edd4f0989e1a765b978fc81 2024-11-14T00:27:57,360 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/5b7478739eaf404c90f51d291112b024 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/5b7478739eaf404c90f51d291112b024 2024-11-14T00:27:57,362 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ac0296d33be44179a3f60ef46febb3f3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ac0296d33be44179a3f60ef46febb3f3 2024-11-14T00:27:57,365 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b2504011c9594d648c7b0380216d608e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/b2504011c9594d648c7b0380216d608e 2024-11-14T00:27:57,367 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ccff9dc510ee4ad4a455e5aa12b5b762 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/C/ccff9dc510ee4ad4a455e5aa12b5b762 2024-11-14T00:27:57,375 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/recovered.edits/407.seqid to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2/recovered.edits/407.seqid 2024-11-14T00:27:57,376 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/285207bf1146874945959d9159bd86b2 2024-11-14T00:27:57,377 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T00:27:57,388 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=25, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:27:57,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-14T00:27:57,402 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T00:27:57,478 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T00:27:57,487 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=25, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:27:57,487 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T00:27:57,487 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731544077487"}]},"ts":"9223372036854775807"} 2024-11-14T00:27:57,503 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T00:27:57,503 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 285207bf1146874945959d9159bd86b2, NAME => 'TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T00:27:57,503 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T00:27:57,503 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731544077503"}]},"ts":"9223372036854775807"} 2024-11-14T00:27:57,508 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T00:27:57,510 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=25, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:27:57,512 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 300 msec 2024-11-14T00:27:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T00:27:57,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T00:27:57,538 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:27:57,556 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 210) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1906822607_22 at /127.0.0.1:38498 [Waiting for operation #205] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;63baa11b75f3:33199-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1495897995_22 at /127.0.0.1:50772 [Waiting for operation #149] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1495897995_22 at /127.0.0.1:50662 [Waiting for operation #192] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1301 (was 1140) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1902 (was 3687) 2024-11-14T00:27:57,573 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=1301, ProcessCount=11, AvailableMemoryMB=1902 2024-11-14T00:27:57,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T00:27:57,576 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T00:27:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T00:27:57,587 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T00:27:57,587 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:57,587 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 26 2024-11-14T00:27:57,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T00:27:57,591 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T00:27:57,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741924_1100 (size=959) 2024-11-14T00:27:57,673 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c 2024-11-14T00:27:57,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741925_1101 (size=53) 2024-11-14T00:27:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T00:27:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T00:27:58,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:27:58,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a2caf5d1c25d111104a7450b3aaca862, disabling compactions & flushes 2024-11-14T00:27:58,090 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:27:58,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:27:58,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. after waiting 0 ms 2024-11-14T00:27:58,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:27:58,090 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:27:58,090 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a2caf5d1c25d111104a7450b3aaca862: Waiting for close lock at 1731544078090Disabling compacts and flushes for region at 1731544078090Disabling writes for close at 1731544078090Writing region close event to WAL at 1731544078090Closed at 1731544078090 2024-11-14T00:27:58,092 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T00:27:58,093 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731544078092"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731544078092"}]},"ts":"1731544078092"} 2024-11-14T00:27:58,095 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T00:27:58,096 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T00:27:58,097 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544078096"}]},"ts":"1731544078096"} 2024-11-14T00:27:58,099 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T00:27:58,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2caf5d1c25d111104a7450b3aaca862, ASSIGN}] 2024-11-14T00:27:58,101 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2caf5d1c25d111104a7450b3aaca862, ASSIGN 2024-11-14T00:27:58,102 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2caf5d1c25d111104a7450b3aaca862, ASSIGN; state=OFFLINE, location=63baa11b75f3,33199,1731544045387; forceNewPlan=false, retain=false 2024-11-14T00:27:58,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T00:27:58,253 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=a2caf5d1c25d111104a7450b3aaca862, regionState=OPENING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:58,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2caf5d1c25d111104a7450b3aaca862, ASSIGN because future has completed 2024-11-14T00:27:58,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:27:58,439 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:27:58,439 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} 2024-11-14T00:27:58,440 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:58,440 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:27:58,440 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:58,440 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:58,442 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:58,458 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2caf5d1c25d111104a7450b3aaca862 columnFamilyName A 2024-11-14T00:27:58,458 DEBUG [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:58,460 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] regionserver.HStore(327): Store=a2caf5d1c25d111104a7450b3aaca862/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:27:58,461 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:58,463 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2caf5d1c25d111104a7450b3aaca862 columnFamilyName B 2024-11-14T00:27:58,463 DEBUG [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:58,463 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] regionserver.HStore(327): Store=a2caf5d1c25d111104a7450b3aaca862/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:27:58,464 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:58,465 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2caf5d1c25d111104a7450b3aaca862 columnFamilyName C 2024-11-14T00:27:58,465 DEBUG [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:58,466 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] regionserver.HStore(327): Store=a2caf5d1c25d111104a7450b3aaca862/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:27:58,466 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:58,467 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:58,468 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:58,469 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:58,469 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:58,469 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T00:27:58,471 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:58,473 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T00:27:58,474 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened a2caf5d1c25d111104a7450b3aaca862; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67802756, jitterRate=0.010339796543121338}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T00:27:58,474 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:58,475 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for a2caf5d1c25d111104a7450b3aaca862: Running coprocessor pre-open hook at 1731544078440Writing region info on filesystem at 1731544078440Initializing all the Stores at 1731544078441 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544078441Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544078441Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544078441Cleaning up temporary data from old regions at 1731544078469 (+28 ms)Running coprocessor post-open hooks at 1731544078474 (+5 ms)Region opened successfully at 1731544078475 (+1 ms) 2024-11-14T00:27:58,476 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., pid=28, masterSystemTime=1731544078435 2024-11-14T00:27:58,485 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:27:58,485 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:27:58,486 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=a2caf5d1c25d111104a7450b3aaca862, regionState=OPEN, openSeqNum=2, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:58,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:27:58,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-14T00:27:58,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 in 227 msec 2024-11-14T00:27:58,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-14T00:27:58,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2caf5d1c25d111104a7450b3aaca862, ASSIGN in 399 msec 2024-11-14T00:27:58,511 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T00:27:58,512 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544078511"}]},"ts":"1731544078511"} 2024-11-14T00:27:58,522 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T00:27:58,524 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T00:27:58,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 948 msec 2024-11-14T00:27:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-14T00:27:58,726 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:27:58,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35bd8b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:58,727 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:58,728 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:58,737 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61274f18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:58,743 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:58,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2df62a25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:58,745 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:58,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:58,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:58,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:58,750 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T00:27:58,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 63baa11b75f3,36271,1731544044347 2024-11-14T00:27:58,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@208889f2 2024-11-14T00:27:58,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T00:27:58,753 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T00:27:58,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T00:27:58,759 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T00:27:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T00:27:58,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741926_1102 (size=995) 2024-11-14T00:27:59,196 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-14T00:27:59,196 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-14T00:27:59,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T00:27:59,211 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2caf5d1c25d111104a7450b3aaca862, REOPEN/MOVE}] 2024-11-14T00:27:59,212 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2caf5d1c25d111104a7450b3aaca862, REOPEN/MOVE 2024-11-14T00:27:59,213 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=a2caf5d1c25d111104a7450b3aaca862, regionState=CLOSING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:59,216 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2caf5d1c25d111104a7450b3aaca862, REOPEN/MOVE because future has completed 2024-11-14T00:27:59,216 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T00:27:59,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:27:59,370 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:59,370 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T00:27:59,370 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing a2caf5d1c25d111104a7450b3aaca862, disabling compactions & flushes 2024-11-14T00:27:59,370 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:27:59,370 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:27:59,370 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. after waiting 0 ms 2024-11-14T00:27:59,370 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:27:59,375 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T00:27:59,376 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:27:59,376 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for a2caf5d1c25d111104a7450b3aaca862: Waiting for close lock at 1731544079370Running coprocessor pre-close hooks at 1731544079370Disabling compacts and flushes for region at 1731544079370Disabling writes for close at 1731544079370Writing region close event to WAL at 1731544079371 (+1 ms)Running coprocessor post-close hooks at 1731544079376 (+5 ms)Closed at 1731544079376 2024-11-14T00:27:59,376 WARN [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionServer(3299): Not adding moved region record: a2caf5d1c25d111104a7450b3aaca862 to self. 2024-11-14T00:27:59,379 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:59,380 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=a2caf5d1c25d111104a7450b3aaca862, regionState=CLOSED 2024-11-14T00:27:59,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:27:59,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-14T00:27:59,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 in 167 msec 2024-11-14T00:27:59,387 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2caf5d1c25d111104a7450b3aaca862, REOPEN/MOVE; state=CLOSED, location=63baa11b75f3,33199,1731544045387; forceNewPlan=false, retain=true 2024-11-14T00:27:59,538 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=a2caf5d1c25d111104a7450b3aaca862, regionState=OPENING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:59,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2caf5d1c25d111104a7450b3aaca862, REOPEN/MOVE because future has completed 2024-11-14T00:27:59,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:27:59,710 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:27:59,710 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} 2024-11-14T00:27:59,710 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:59,711 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:27:59,711 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:59,711 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:59,715 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:59,722 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2caf5d1c25d111104a7450b3aaca862 columnFamilyName A 2024-11-14T00:27:59,728 DEBUG [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:59,729 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] regionserver.HStore(327): Store=a2caf5d1c25d111104a7450b3aaca862/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:27:59,729 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:59,730 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2caf5d1c25d111104a7450b3aaca862 columnFamilyName B 2024-11-14T00:27:59,731 DEBUG [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:59,731 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] regionserver.HStore(327): Store=a2caf5d1c25d111104a7450b3aaca862/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:27:59,731 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:59,733 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2caf5d1c25d111104a7450b3aaca862 columnFamilyName C 2024-11-14T00:27:59,733 DEBUG [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:27:59,735 INFO [StoreOpener-a2caf5d1c25d111104a7450b3aaca862-1 {}] regionserver.HStore(327): Store=a2caf5d1c25d111104a7450b3aaca862/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:27:59,735 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:59,737 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:59,738 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:59,746 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:59,746 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:59,747 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T00:27:59,751 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:59,753 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened a2caf5d1c25d111104a7450b3aaca862; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65310839, jitterRate=-0.026792660355567932}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T00:27:59,754 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:27:59,755 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for a2caf5d1c25d111104a7450b3aaca862: Running coprocessor pre-open hook at 1731544079711Writing region info on filesystem at 1731544079711Initializing all the Stores at 1731544079712 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544079712Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544079712Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544079712Cleaning up temporary data from old regions at 1731544079746 (+34 ms)Running coprocessor post-open hooks at 1731544079754 (+8 ms)Region opened successfully at 1731544079755 (+1 ms) 2024-11-14T00:27:59,757 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., pid=33, masterSystemTime=1731544079706 2024-11-14T00:27:59,767 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:27:59,767 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:27:59,768 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=a2caf5d1c25d111104a7450b3aaca862, regionState=OPEN, openSeqNum=5, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:27:59,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:27:59,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=31 2024-11-14T00:27:59,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 in 231 msec 2024-11-14T00:27:59,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-14T00:27:59,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2caf5d1c25d111104a7450b3aaca862, REOPEN/MOVE in 583 msec 2024-11-14T00:27:59,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-14T00:27:59,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 597 msec 2024-11-14T00:27:59,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0440 sec 2024-11-14T00:27:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-14T00:27:59,811 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:27:59,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca7569, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:59,821 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:59,825 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d7b64f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:59,827 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d1de7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a40bbe7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:59,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:59,838 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32bb8111, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:59,841 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c811936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63f4ae4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,852 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:59,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:59,856 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:59,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f225768, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:59,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:59,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:59,865 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:59,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f7544d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf9a2c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:59,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:59,873 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db8c981, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:59,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:59,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:59,877 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:59,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42b37c5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3760dda8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:59,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:59,885 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a098cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:59,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:59,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:59,889 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:59,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ab8f21a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40ce1eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:59,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:59,907 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a366016, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:59,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:59,920 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:59,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af3f814, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,924 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16bb65dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,924 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:59,924 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:59,926 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54cfb6c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:59,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:59,929 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c58612, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70861d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:59,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:59,934 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:59,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:59,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69741439, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:59,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:59,937 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:59,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7568872a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61db675, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:27:59,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:27:59,945 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:27:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:27:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:27:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e840344, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,945 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:27:59,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:27:59,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:27:59,947 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:27:59,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9f9fc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:27:59,949 DEBUG [Thread-541 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:59,950 DEBUG [Thread-550 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:59,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:59,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:59,952 DEBUG [Thread-552 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:59,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:59,953 DEBUG [Thread-543 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:59,953 DEBUG [Thread-539 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:59,954 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:59,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:59,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:59,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:59,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:59,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:59,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:59,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:59,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:59,960 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:27:59,961 DEBUG [Thread-554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-14T00:27:59,961 DEBUG [Thread-548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:59,962 DEBUG [Thread-537 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:59,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:59,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:59,964 DEBUG [Thread-545 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:27:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:27:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T00:27:59,976 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:27:59,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:59,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:27:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:27:59,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:27:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:27:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:27:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:27:59,980 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:59,981 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:59,983 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:27:59,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:27:59,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:27:59,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:27:59,986 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:27:59,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:27:59,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:28:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:00,018 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:00,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T00:28:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T00:28:00,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c415cd38326b4d25a46ad622359c6fad_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544080011/Put/seqid=0 2024-11-14T00:28:00,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741927_1103 (size=12154) 2024-11-14T00:28:00,142 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:00,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T00:28:00,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:00,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:00,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:00,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,186 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c415cd38326b4d25a46ad622359c6fad_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c415cd38326b4d25a46ad622359c6fad_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:00,191 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/13afcc7f0d364fd5acf02a68c2e7a634, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:00,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/13afcc7f0d364fd5acf02a68c2e7a634 is 175, key is test_row_0/A:col10/1731544080011/Put/seqid=0 2024-11-14T00:28:00,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741928_1104 (size=30955) 2024-11-14T00:28:00,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544140233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544140239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544140249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T00:28:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544140289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T00:28:00,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:00,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:00,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:00,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544140286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544140396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544140406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544140406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544140408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544140416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T00:28:00,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:00,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:00,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:00,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T00:28:00,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544140606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T00:28:00,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:00,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:00,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:00,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544140625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544140625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544140625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544140636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,690 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/13afcc7f0d364fd5acf02a68c2e7a634 2024-11-14T00:28:00,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/540d2c66e0024fbfae450647e5deb7c5 is 50, key is test_row_0/B:col10/1731544080011/Put/seqid=0 2024-11-14T00:28:00,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T00:28:00,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:00,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:00,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:00,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741929_1105 (size=12001) 2024-11-14T00:28:00,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/540d2c66e0024fbfae450647e5deb7c5 2024-11-14T00:28:00,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/62c8077ff9e14fa387ae99596c19acd4 is 50, key is test_row_0/C:col10/1731544080011/Put/seqid=0 2024-11-14T00:28:00,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741930_1106 (size=12001) 2024-11-14T00:28:00,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/62c8077ff9e14fa387ae99596c19acd4 2024-11-14T00:28:00,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544140919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544140935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544140936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T00:28:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:00,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:00,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:00,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/13afcc7f0d364fd5acf02a68c2e7a634 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/13afcc7f0d364fd5acf02a68c2e7a634 2024-11-14T00:28:00,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544140938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:00,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544140956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:00,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:00,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/13afcc7f0d364fd5acf02a68c2e7a634, entries=150, sequenceid=17, filesize=30.2 K 2024-11-14T00:28:00,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/540d2c66e0024fbfae450647e5deb7c5 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/540d2c66e0024fbfae450647e5deb7c5 2024-11-14T00:28:01,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/540d2c66e0024fbfae450647e5deb7c5, entries=150, sequenceid=17, filesize=11.7 K 2024-11-14T00:28:01,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/62c8077ff9e14fa387ae99596c19acd4 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/62c8077ff9e14fa387ae99596c19acd4 2024-11-14T00:28:01,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/62c8077ff9e14fa387ae99596c19acd4, entries=150, sequenceid=17, filesize=11.7 K 2024-11-14T00:28:01,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for a2caf5d1c25d111104a7450b3aaca862 in 1066ms, sequenceid=17, compaction requested=false 2024-11-14T00:28:01,084 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T00:28:01,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:01,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-14T00:28:01,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:01,105 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T00:28:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T00:28:01,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111465f4572b9f3247f486422735afdbc1fa_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544080278/Put/seqid=0 2024-11-14T00:28:01,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741931_1107 (size=12154) 2024-11-14T00:28:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:01,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:01,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:01,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544141474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544141473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:01,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544141475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:01,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544141480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544141480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:01,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:01,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544141585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:01,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544141588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:01,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:01,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:01,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:01,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544141597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:01,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544141602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:01,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:01,629 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111465f4572b9f3247f486422735afdbc1fa_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111465f4572b9f3247f486422735afdbc1fa_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:01,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/2013b42656da4f5997614b8fd081a0af, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:01,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/2013b42656da4f5997614b8fd081a0af is 175, key is test_row_0/A:col10/1731544080278/Put/seqid=0 2024-11-14T00:28:01,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741932_1108 (size=30955) 2024-11-14T00:28:01,691 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/2013b42656da4f5997614b8fd081a0af 2024-11-14T00:28:01,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/6a287f1466f04b12bdc14b24d1086ee4 is 50, key is test_row_0/B:col10/1731544080278/Put/seqid=0 2024-11-14T00:28:01,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741933_1109 (size=12001) 2024-11-14T00:28:01,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544141817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:01,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:01,828 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T00:28:01,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544141820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:01,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544141821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:01,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544141827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:01,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:01,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:02,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544142125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T00:28:02,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:02,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544142136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:02,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544142136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:02,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:02,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544142145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:02,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:02,174 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/6a287f1466f04b12bdc14b24d1086ee4 2024-11-14T00:28:02,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/f20a76de5d7a4dc4b528b092efa803ef is 50, key is test_row_0/C:col10/1731544080278/Put/seqid=0 2024-11-14T00:28:02,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741934_1110 (size=12001) 2024-11-14T00:28:02,240 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/f20a76de5d7a4dc4b528b092efa803ef 2024-11-14T00:28:02,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/2013b42656da4f5997614b8fd081a0af as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/2013b42656da4f5997614b8fd081a0af 2024-11-14T00:28:02,276 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/2013b42656da4f5997614b8fd081a0af, entries=150, sequenceid=41, filesize=30.2 K 2024-11-14T00:28:02,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/6a287f1466f04b12bdc14b24d1086ee4 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/6a287f1466f04b12bdc14b24d1086ee4 2024-11-14T00:28:02,291 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/6a287f1466f04b12bdc14b24d1086ee4, entries=150, sequenceid=41, filesize=11.7 K 2024-11-14T00:28:02,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/f20a76de5d7a4dc4b528b092efa803ef as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/f20a76de5d7a4dc4b528b092efa803ef 2024-11-14T00:28:02,305 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/f20a76de5d7a4dc4b528b092efa803ef, entries=150, sequenceid=41, filesize=11.7 K 2024-11-14T00:28:02,307 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for a2caf5d1c25d111104a7450b3aaca862 in 1202ms, sequenceid=41, compaction requested=false 2024-11-14T00:28:02,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:02,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:02,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-14T00:28:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-14T00:28:02,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-14T00:28:02,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3260 sec 2024-11-14T00:28:02,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 2.3550 sec 2024-11-14T00:28:02,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:02,501 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:02,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T00:28:02,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111494d0ada3fce048888c56256a8ba1baa9_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544082496/Put/seqid=0 2024-11-14T00:28:02,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741935_1111 (size=14594) 2024-11-14T00:28:02,588 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:02,595 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111494d0ada3fce048888c56256a8ba1baa9_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111494d0ada3fce048888c56256a8ba1baa9_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:02,597 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/2aa166c0c5354ec6a747d5ee53bc8e8c, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:02,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/2aa166c0c5354ec6a747d5ee53bc8e8c is 175, key is test_row_0/A:col10/1731544082496/Put/seqid=0 2024-11-14T00:28:02,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741936_1112 (size=39549) 2024-11-14T00:28:02,658 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/2aa166c0c5354ec6a747d5ee53bc8e8c 2024-11-14T00:28:02,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/4548daece36f4185b46b882ad74a44d3 is 50, key is test_row_0/B:col10/1731544082496/Put/seqid=0 2024-11-14T00:28:02,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:02,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544142722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:02,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544142724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:02,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:02,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544142729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:02,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:02,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:02,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544142734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:02,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544142733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:02,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:02,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:02,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741937_1113 (size=12001) 2024-11-14T00:28:02,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544142840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:02,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544142856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:02,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544142856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:02,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544142856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:02,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:02,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544142856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:02,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:02,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:02,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:03,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544143057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:03,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544143076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:03,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544143076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:03,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544143076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:03,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544143077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:03,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:03,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:03,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:03,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/4548daece36f4185b46b882ad74a44d3 2024-11-14T00:28:03,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/20b5650261414f62a154d4656aa107d7 is 50, key is test_row_0/C:col10/1731544082496/Put/seqid=0 2024-11-14T00:28:03,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741938_1114 (size=12001) 2024-11-14T00:28:03,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/20b5650261414f62a154d4656aa107d7 2024-11-14T00:28:03,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/2aa166c0c5354ec6a747d5ee53bc8e8c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/2aa166c0c5354ec6a747d5ee53bc8e8c 2024-11-14T00:28:03,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/2aa166c0c5354ec6a747d5ee53bc8e8c, entries=200, sequenceid=55, filesize=38.6 K 2024-11-14T00:28:03,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/4548daece36f4185b46b882ad74a44d3 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/4548daece36f4185b46b882ad74a44d3 2024-11-14T00:28:03,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/4548daece36f4185b46b882ad74a44d3, entries=150, sequenceid=55, filesize=11.7 K 2024-11-14T00:28:03,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/20b5650261414f62a154d4656aa107d7 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/20b5650261414f62a154d4656aa107d7 2024-11-14T00:28:03,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544143365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:03,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544143388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:03,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:03,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:03,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544143388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:03,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544143395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:03,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:03,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544143395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:03,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:03,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:03,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/20b5650261414f62a154d4656aa107d7, entries=150, sequenceid=55, filesize=11.7 K 2024-11-14T00:28:03,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for a2caf5d1c25d111104a7450b3aaca862 in 947ms, sequenceid=55, compaction requested=true 2024-11-14T00:28:03,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:03,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:03,449 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:03,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:03,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:03,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:03,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:03,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:28:03,452 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:03,465 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:03,465 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/A is initiating minor compaction (all files) 2024-11-14T00:28:03,465 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/A in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:03,465 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/13afcc7f0d364fd5acf02a68c2e7a634, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/2013b42656da4f5997614b8fd081a0af, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/2aa166c0c5354ec6a747d5ee53bc8e8c] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=99.1 K 2024-11-14T00:28:03,465 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:03,465 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/13afcc7f0d364fd5acf02a68c2e7a634, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/2013b42656da4f5997614b8fd081a0af, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/2aa166c0c5354ec6a747d5ee53bc8e8c] 2024-11-14T00:28:03,466 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:03,466 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/B is initiating minor compaction (all files) 2024-11-14T00:28:03,466 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/B in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:03,466 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/540d2c66e0024fbfae450647e5deb7c5, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/6a287f1466f04b12bdc14b24d1086ee4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/4548daece36f4185b46b882ad74a44d3] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=35.2 K 2024-11-14T00:28:03,467 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 540d2c66e0024fbfae450647e5deb7c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731544080005 2024-11-14T00:28:03,467 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13afcc7f0d364fd5acf02a68c2e7a634, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731544080005 2024-11-14T00:28:03,467 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a287f1466f04b12bdc14b24d1086ee4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731544080177 2024-11-14T00:28:03,467 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2013b42656da4f5997614b8fd081a0af, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731544080177 2024-11-14T00:28:03,468 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 4548daece36f4185b46b882ad74a44d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731544081474 2024-11-14T00:28:03,468 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2aa166c0c5354ec6a747d5ee53bc8e8c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731544081464 2024-11-14T00:28:03,504 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#B#compaction#96 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:03,505 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/41c157b8a7574bd49a5996b806019a6c is 50, key is test_row_0/B:col10/1731544082496/Put/seqid=0 2024-11-14T00:28:03,513 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:03,554 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111422ece4d47b094f3fa76b8bf9ea99da4c_a2caf5d1c25d111104a7450b3aaca862 store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:03,559 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111422ece4d47b094f3fa76b8bf9ea99da4c_a2caf5d1c25d111104a7450b3aaca862, store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:03,559 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111422ece4d47b094f3fa76b8bf9ea99da4c_a2caf5d1c25d111104a7450b3aaca862 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:03,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741939_1115 (size=12104) 2024-11-14T00:28:03,584 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/41c157b8a7574bd49a5996b806019a6c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/41c157b8a7574bd49a5996b806019a6c 2024-11-14T00:28:03,591 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/B of a2caf5d1c25d111104a7450b3aaca862 into 41c157b8a7574bd49a5996b806019a6c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:03,591 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:03,591 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/B, priority=13, startTime=1731544083449; duration=0sec 2024-11-14T00:28:03,591 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:03,591 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:B 2024-11-14T00:28:03,591 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:03,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741940_1116 (size=4469) 2024-11-14T00:28:03,593 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:03,593 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/C is initiating minor compaction (all files) 2024-11-14T00:28:03,593 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/C in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:03,593 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/62c8077ff9e14fa387ae99596c19acd4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/f20a76de5d7a4dc4b528b092efa803ef, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/20b5650261414f62a154d4656aa107d7] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=35.2 K 2024-11-14T00:28:03,600 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 62c8077ff9e14fa387ae99596c19acd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731544080005 2024-11-14T00:28:03,601 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting f20a76de5d7a4dc4b528b092efa803ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731544080177 2024-11-14T00:28:03,601 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 20b5650261414f62a154d4656aa107d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731544081474 2024-11-14T00:28:03,604 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#A#compaction#97 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:03,605 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/ea59b97f2d7d412983079a617f623629 is 175, key is test_row_0/A:col10/1731544082496/Put/seqid=0 2024-11-14T00:28:03,627 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#C#compaction#98 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:03,627 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/9e8ca91cabed42d4a7117aac5efca04f is 50, key is test_row_0/C:col10/1731544082496/Put/seqid=0 2024-11-14T00:28:03,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741941_1117 (size=31058) 2024-11-14T00:28:03,669 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/ea59b97f2d7d412983079a617f623629 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/ea59b97f2d7d412983079a617f623629 2024-11-14T00:28:03,685 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/A of a2caf5d1c25d111104a7450b3aaca862 into ea59b97f2d7d412983079a617f623629(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:03,685 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:03,685 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/A, priority=13, startTime=1731544083449; duration=0sec 2024-11-14T00:28:03,686 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:03,686 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:A 2024-11-14T00:28:03,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741942_1118 (size=12104) 2024-11-14T00:28:03,750 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/9e8ca91cabed42d4a7117aac5efca04f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/9e8ca91cabed42d4a7117aac5efca04f 2024-11-14T00:28:03,768 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/C of a2caf5d1c25d111104a7450b3aaca862 into 9e8ca91cabed42d4a7117aac5efca04f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:03,768 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:03,768 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/C, priority=13, startTime=1731544083449; duration=0sec 2024-11-14T00:28:03,768 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:03,768 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:C 2024-11-14T00:28:03,926 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:03,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T00:28:03,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:03,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144e20784905d74ea9b8ad614c856ed8ce_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544083924/Put/seqid=0 2024-11-14T00:28:03,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544143960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:03,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544143961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:03,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:03,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:03,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544143977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:03,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544143981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:03,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:03,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:03,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544143988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741943_1119 (size=19474) 2024-11-14T00:28:04,011 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:04,020 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144e20784905d74ea9b8ad614c856ed8ce_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144e20784905d74ea9b8ad614c856ed8ce_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:04,022 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/24d02dd4913f4556b9e71dc8f72b514f, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:04,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/24d02dd4913f4556b9e71dc8f72b514f is 175, key is test_row_0/A:col10/1731544083924/Put/seqid=0 2024-11-14T00:28:04,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741944_1120 (size=56733) 2024-11-14T00:28:04,073 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/24d02dd4913f4556b9e71dc8f72b514f 2024-11-14T00:28:04,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/b6697dd5f67e473f9857d299f8874a71 is 50, key is test_row_0/B:col10/1731544083924/Put/seqid=0 2024-11-14T00:28:04,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544144082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544144095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544144096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544144085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544144106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741945_1121 (size=12001) 2024-11-14T00:28:04,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/b6697dd5f67e473f9857d299f8874a71 2024-11-14T00:28:04,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-14T00:28:04,157 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:04,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-14T00:28:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T00:28:04,164 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:04,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/c677356e016c44c08692215b3cc08b0b is 50, key is test_row_0/C:col10/1731544083924/Put/seqid=0 2024-11-14T00:28:04,166 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:04,166 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:04,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741946_1122 (size=12001) 2024-11-14T00:28:04,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/c677356e016c44c08692215b3cc08b0b 2024-11-14T00:28:04,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/24d02dd4913f4556b9e71dc8f72b514f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/24d02dd4913f4556b9e71dc8f72b514f 2024-11-14T00:28:04,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/24d02dd4913f4556b9e71dc8f72b514f, entries=300, sequenceid=84, filesize=55.4 K 2024-11-14T00:28:04,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/b6697dd5f67e473f9857d299f8874a71 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/b6697dd5f67e473f9857d299f8874a71 2024-11-14T00:28:04,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/b6697dd5f67e473f9857d299f8874a71, entries=150, sequenceid=84, filesize=11.7 K 2024-11-14T00:28:04,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/c677356e016c44c08692215b3cc08b0b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/c677356e016c44c08692215b3cc08b0b 2024-11-14T00:28:04,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/c677356e016c44c08692215b3cc08b0b, entries=150, sequenceid=84, filesize=11.7 K 2024-11-14T00:28:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T00:28:04,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for a2caf5d1c25d111104a7450b3aaca862 in 354ms, sequenceid=84, compaction requested=false 2024-11-14T00:28:04,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:04,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:04,309 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:04,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T00:28:04,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T00:28:04,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:04,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:04,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:04,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:04,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:04,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d272dcb7330c4f4b962976ceee92d03c_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544083966/Put/seqid=0 2024-11-14T00:28:04,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741947_1123 (size=14594) 2024-11-14T00:28:04,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544144398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544144402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544144410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544144413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544144417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,424 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T00:28:04,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:04,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:04,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:04,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:04,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T00:28:04,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544144515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544144526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544144526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,527 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544144526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544144536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T00:28:04,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:04,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:04,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:04,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:04,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:04,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544144728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544144737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544144737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544144738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:04,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544144757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:04,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:04,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:04,776 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:04,786 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d272dcb7330c4f4b962976ceee92d03c_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d272dcb7330c4f4b962976ceee92d03c_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:04,788 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/00461c02e34748e58ff997c9c14664d1, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:04,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/00461c02e34748e58ff997c9c14664d1 is 175, key is test_row_0/A:col10/1731544083966/Put/seqid=0 2024-11-14T00:28:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T00:28:04,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T00:28:04,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:04,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:04,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:04,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:04,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:04,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:04,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741948_1124 (size=39549) 2024-11-14T00:28:04,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T00:28:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:04,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:04,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:04,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:05,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544145046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544145056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544145056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544145056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544145076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T00:28:05,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:05,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:05,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:05,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:05,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:05,247 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/00461c02e34748e58ff997c9c14664d1 2024-11-14T00:28:05,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/655233bce130438da2617761bdbe3d2c is 50, key is test_row_0/B:col10/1731544083966/Put/seqid=0 2024-11-14T00:28:05,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T00:28:05,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:05,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:05,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:05,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:05,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T00:28:05,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741949_1125 (size=12001) 2024-11-14T00:28:05,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/655233bce130438da2617761bdbe3d2c 2024-11-14T00:28:05,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/f52791da0f054bed9c03d789ebac8ffe is 50, key is test_row_0/C:col10/1731544083966/Put/seqid=0 2024-11-14T00:28:05,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741950_1126 (size=12001) 2024-11-14T00:28:05,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/f52791da0f054bed9c03d789ebac8ffe 2024-11-14T00:28:05,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/00461c02e34748e58ff997c9c14664d1 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/00461c02e34748e58ff997c9c14664d1 2024-11-14T00:28:05,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/00461c02e34748e58ff997c9c14664d1, entries=200, sequenceid=97, filesize=38.6 K 2024-11-14T00:28:05,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/655233bce130438da2617761bdbe3d2c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/655233bce130438da2617761bdbe3d2c 2024-11-14T00:28:05,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/655233bce130438da2617761bdbe3d2c, entries=150, sequenceid=97, filesize=11.7 K 2024-11-14T00:28:05,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/f52791da0f054bed9c03d789ebac8ffe as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/f52791da0f054bed9c03d789ebac8ffe 2024-11-14T00:28:05,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/f52791da0f054bed9c03d789ebac8ffe, entries=150, sequenceid=97, filesize=11.7 K 2024-11-14T00:28:05,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for a2caf5d1c25d111104a7450b3aaca862 in 1103ms, sequenceid=97, compaction requested=true 2024-11-14T00:28:05,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:05,412 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:05,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:05,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:05,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:05,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:05,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:05,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:28:05,415 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:05,416 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127340 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:05,416 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/A is initiating minor compaction (all files) 2024-11-14T00:28:05,417 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/A in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:05,417 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/ea59b97f2d7d412983079a617f623629, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/24d02dd4913f4556b9e71dc8f72b514f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/00461c02e34748e58ff997c9c14664d1] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=124.4 K 2024-11-14T00:28:05,417 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:05,417 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/ea59b97f2d7d412983079a617f623629, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/24d02dd4913f4556b9e71dc8f72b514f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/00461c02e34748e58ff997c9c14664d1] 2024-11-14T00:28:05,418 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea59b97f2d7d412983079a617f623629, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731544081474 2024-11-14T00:28:05,419 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24d02dd4913f4556b9e71dc8f72b514f, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731544082710 2024-11-14T00:28:05,420 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00461c02e34748e58ff997c9c14664d1, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731544083945 2024-11-14T00:28:05,427 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:05,428 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/B is initiating minor compaction (all files) 2024-11-14T00:28:05,428 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/B in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:05,428 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/41c157b8a7574bd49a5996b806019a6c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/b6697dd5f67e473f9857d299f8874a71, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/655233bce130438da2617761bdbe3d2c] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=35.3 K 2024-11-14T00:28:05,433 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 41c157b8a7574bd49a5996b806019a6c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731544081474 2024-11-14T00:28:05,435 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting b6697dd5f67e473f9857d299f8874a71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731544082710 2024-11-14T00:28:05,436 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 655233bce130438da2617761bdbe3d2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731544083961 2024-11-14T00:28:05,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-14T00:28:05,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:05,438 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T00:28:05,463 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:05,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111403ed005ec14d42eb97d04ba856406694_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544084368/Put/seqid=0 2024-11-14T00:28:05,498 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#B#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:05,499 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/b28de671374443acb15e8ab0174e46c1 is 50, key is test_row_0/B:col10/1731544083966/Put/seqid=0 2024-11-14T00:28:05,501 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411147398fa285eb0465786ba82208d4d6981_a2caf5d1c25d111104a7450b3aaca862 store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:05,505 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411147398fa285eb0465786ba82208d4d6981_a2caf5d1c25d111104a7450b3aaca862, store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:05,505 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147398fa285eb0465786ba82208d4d6981_a2caf5d1c25d111104a7450b3aaca862 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:05,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741951_1127 (size=12154) 2024-11-14T00:28:05,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:05,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:05,589 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111403ed005ec14d42eb97d04ba856406694_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111403ed005ec14d42eb97d04ba856406694_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:05,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/04982b8567c842679a294094f9913038, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:05,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/04982b8567c842679a294094f9913038 is 175, key is test_row_0/A:col10/1731544084368/Put/seqid=0 2024-11-14T00:28:05,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741952_1128 (size=12207) 2024-11-14T00:28:05,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741953_1129 (size=4469) 2024-11-14T00:28:05,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544145605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544145611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544145612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,642 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/b28de671374443acb15e8ab0174e46c1 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/b28de671374443acb15e8ab0174e46c1 2024-11-14T00:28:05,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544145639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544145622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741954_1130 (size=30955) 2024-11-14T00:28:05,651 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/04982b8567c842679a294094f9913038 2024-11-14T00:28:05,659 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/B of a2caf5d1c25d111104a7450b3aaca862 into b28de671374443acb15e8ab0174e46c1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:05,659 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:05,659 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/B, priority=13, startTime=1731544085414; duration=0sec 2024-11-14T00:28:05,659 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:05,659 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:B 2024-11-14T00:28:05,659 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:05,663 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:05,663 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/C is initiating minor compaction (all files) 2024-11-14T00:28:05,663 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/C in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:05,663 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/9e8ca91cabed42d4a7117aac5efca04f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/c677356e016c44c08692215b3cc08b0b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/f52791da0f054bed9c03d789ebac8ffe] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=35.3 K 2024-11-14T00:28:05,664 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e8ca91cabed42d4a7117aac5efca04f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731544081474 2024-11-14T00:28:05,667 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c677356e016c44c08692215b3cc08b0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731544082710 2024-11-14T00:28:05,670 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting f52791da0f054bed9c03d789ebac8ffe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731544083961 2024-11-14T00:28:05,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/d1c02a080fc24518b7f26e37922a7bfe is 50, key is test_row_0/B:col10/1731544084368/Put/seqid=0 2024-11-14T00:28:05,722 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#C#compaction#109 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:05,723 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/b6272871c2b54c6b8cd78203204905c5 is 50, key is test_row_0/C:col10/1731544083966/Put/seqid=0 2024-11-14T00:28:05,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544145735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544145736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544145746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544145753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544145756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741955_1131 (size=12001) 2024-11-14T00:28:05,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741956_1132 (size=12207) 2024-11-14T00:28:05,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544145955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544145955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544145968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544145975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:05,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544145975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:05,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:05,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:06,030 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#A#compaction#105 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:06,030 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/8eb547e24e8b4e46bc91139f917b5763 is 175, key is test_row_0/A:col10/1731544083966/Put/seqid=0 2024-11-14T00:28:06,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741957_1133 (size=31161) 2024-11-14T00:28:06,188 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/d1c02a080fc24518b7f26e37922a7bfe 2024-11-14T00:28:06,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/0f3a39220fe340dfac55a9c9109bf621 is 50, key is test_row_0/C:col10/1731544084368/Put/seqid=0 2024-11-14T00:28:06,229 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/b6272871c2b54c6b8cd78203204905c5 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/b6272871c2b54c6b8cd78203204905c5 2024-11-14T00:28:06,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741958_1134 (size=12001) 2024-11-14T00:28:06,241 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/C of a2caf5d1c25d111104a7450b3aaca862 into b6272871c2b54c6b8cd78203204905c5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:06,241 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:06,241 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/C, priority=13, startTime=1731544085414; duration=0sec 2024-11-14T00:28:06,241 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:06,241 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:C 2024-11-14T00:28:06,242 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/0f3a39220fe340dfac55a9c9109bf621 2024-11-14T00:28:06,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/04982b8567c842679a294094f9913038 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/04982b8567c842679a294094f9913038 2024-11-14T00:28:06,267 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/04982b8567c842679a294094f9913038, entries=150, sequenceid=122, filesize=30.2 K 2024-11-14T00:28:06,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/d1c02a080fc24518b7f26e37922a7bfe as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/d1c02a080fc24518b7f26e37922a7bfe 2024-11-14T00:28:06,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:06,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544146276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:06,281 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/d1c02a080fc24518b7f26e37922a7bfe, entries=150, sequenceid=122, filesize=11.7 K 2024-11-14T00:28:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/0f3a39220fe340dfac55a9c9109bf621 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/0f3a39220fe340dfac55a9c9109bf621 2024-11-14T00:28:06,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:06,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544146277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:06,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:06,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544146286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:06,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:06,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:06,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544146286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:06,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:06,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544146286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:06,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:06,296 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/0f3a39220fe340dfac55a9c9109bf621, entries=150, sequenceid=122, filesize=11.7 K 2024-11-14T00:28:06,299 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a2caf5d1c25d111104a7450b3aaca862 in 861ms, sequenceid=122, compaction requested=false 2024-11-14T00:28:06,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:06,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:06,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-14T00:28:06,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-14T00:28:06,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-14T00:28:06,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1360 sec 2024-11-14T00:28:06,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.1470 sec 2024-11-14T00:28:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-14T00:28:06,329 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:06,333 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:06,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-14T00:28:06,336 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:06,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T00:28:06,338 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:06,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T00:28:06,483 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/8eb547e24e8b4e46bc91139f917b5763 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/8eb547e24e8b4e46bc91139f917b5763 2024-11-14T00:28:06,495 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/A of a2caf5d1c25d111104a7450b3aaca862 into 8eb547e24e8b4e46bc91139f917b5763(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-14T00:28:06,496 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:06,496 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/A, priority=13, startTime=1731544085412; duration=1sec 2024-11-14T00:28:06,496 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:06,496 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:A 2024-11-14T00:28:06,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T00:28:06,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:06,497 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T00:28:06,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a534db2733cd4406acd3d8b4843e4d3a_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544085612/Put/seqid=0 2024-11-14T00:28:06,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741959_1135 (size=12304) 2024-11-14T00:28:06,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:06,605 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a534db2733cd4406acd3d8b4843e4d3a_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a534db2733cd4406acd3d8b4843e4d3a_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:06,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/244312e34c6045a3a144cf1d04f66e56, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:06,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/244312e34c6045a3a144cf1d04f66e56 is 175, key is test_row_0/A:col10/1731544085612/Put/seqid=0 2024-11-14T00:28:06,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T00:28:06,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741960_1136 (size=31105) 2024-11-14T00:28:06,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:06,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:06,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:06,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544146879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:06,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:06,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544146890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:06,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:06,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544146890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:06,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:06,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:06,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:06,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544146892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:06,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:06,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:06,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544146897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:06,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:06,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:06,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T00:28:07,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544146998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:07,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:07,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544147017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:07,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544147018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:07,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544147018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:07,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544147018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:07,085 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/244312e34c6045a3a144cf1d04f66e56 2024-11-14T00:28:07,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/f7e47a0721f6452b863caf20e3c8d8b0 is 50, key is test_row_0/B:col10/1731544085612/Put/seqid=0 2024-11-14T00:28:07,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741961_1137 (size=12151) 2024-11-14T00:28:07,183 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/f7e47a0721f6452b863caf20e3c8d8b0 2024-11-14T00:28:07,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/e6c59268de854064bdf359fdac5e2493 is 50, key is test_row_0/C:col10/1731544085612/Put/seqid=0 2024-11-14T00:28:07,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:07,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544147220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:07,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741962_1138 (size=12151) 2024-11-14T00:28:07,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544147242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:07,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544147242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:07,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544147242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:07,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544147248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:07,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T00:28:07,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544147535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:07,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:07,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544147565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:07,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544147565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544147566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:07,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544147568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:07,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:07,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:07,657 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/e6c59268de854064bdf359fdac5e2493 2024-11-14T00:28:07,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/244312e34c6045a3a144cf1d04f66e56 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/244312e34c6045a3a144cf1d04f66e56 2024-11-14T00:28:07,685 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/244312e34c6045a3a144cf1d04f66e56, entries=150, sequenceid=136, filesize=30.4 K 2024-11-14T00:28:07,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/f7e47a0721f6452b863caf20e3c8d8b0 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/f7e47a0721f6452b863caf20e3c8d8b0 2024-11-14T00:28:07,712 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/f7e47a0721f6452b863caf20e3c8d8b0, entries=150, sequenceid=136, filesize=11.9 K 2024-11-14T00:28:07,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/e6c59268de854064bdf359fdac5e2493 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/e6c59268de854064bdf359fdac5e2493 2024-11-14T00:28:07,736 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/e6c59268de854064bdf359fdac5e2493, entries=150, sequenceid=136, filesize=11.9 K 2024-11-14T00:28:07,737 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a2caf5d1c25d111104a7450b3aaca862 in 1240ms, sequenceid=136, compaction requested=true 2024-11-14T00:28:07,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:07,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:07,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-14T00:28:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-14T00:28:07,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-14T00:28:07,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4020 sec 2024-11-14T00:28:07,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.4220 sec 2024-11-14T00:28:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:08,064 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:08,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T00:28:08,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142ec3de2d84584712acabbab0259df0cc_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544086886/Put/seqid=0 2024-11-14T00:28:08,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544148096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544148096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544148101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544148116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544148118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741963_1139 (size=14794) 2024-11-14T00:28:08,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544148225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544148225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544148225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544148236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544148236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544148435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544148435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544148436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544148455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544148455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T00:28:08,486 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:08,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-14T00:28:08,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T00:28:08,490 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:08,492 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:08,492 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:08,545 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:08,551 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142ec3de2d84584712acabbab0259df0cc_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142ec3de2d84584712acabbab0259df0cc_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:08,555 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/fe4ad4171d2849b08a4d862c6766661a, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:08,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/fe4ad4171d2849b08a4d862c6766661a is 175, key is test_row_0/A:col10/1731544086886/Put/seqid=0 2024-11-14T00:28:08,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741964_1140 (size=39749) 2024-11-14T00:28:08,582 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/fe4ad4171d2849b08a4d862c6766661a 2024-11-14T00:28:08,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T00:28:08,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/4fcd0c8bbccf4707af545ee965a58ae8 is 50, key is test_row_0/B:col10/1731544086886/Put/seqid=0 2024-11-14T00:28:08,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741965_1141 (size=12151) 2024-11-14T00:28:08,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T00:28:08,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/4fcd0c8bbccf4707af545ee965a58ae8 2024-11-14T00:28:08,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:08,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:08,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:08,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:08,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:08,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/64092ab7132d421ab422eef2f9f5d546 is 50, key is test_row_0/C:col10/1731544086886/Put/seqid=0 2024-11-14T00:28:08,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741966_1142 (size=12151) 2024-11-14T00:28:08,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/64092ab7132d421ab422eef2f9f5d546 2024-11-14T00:28:08,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/fe4ad4171d2849b08a4d862c6766661a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/fe4ad4171d2849b08a4d862c6766661a 2024-11-14T00:28:08,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/fe4ad4171d2849b08a4d862c6766661a, entries=200, sequenceid=162, filesize=38.8 K 2024-11-14T00:28:08,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/4fcd0c8bbccf4707af545ee965a58ae8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/4fcd0c8bbccf4707af545ee965a58ae8 2024-11-14T00:28:08,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/4fcd0c8bbccf4707af545ee965a58ae8, entries=150, sequenceid=162, filesize=11.9 K 2024-11-14T00:28:08,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/64092ab7132d421ab422eef2f9f5d546 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/64092ab7132d421ab422eef2f9f5d546 2024-11-14T00:28:08,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544148756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/64092ab7132d421ab422eef2f9f5d546, entries=150, sequenceid=162, filesize=11.9 K 2024-11-14T00:28:08,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544148756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for a2caf5d1c25d111104a7450b3aaca862 in 701ms, sequenceid=162, compaction requested=true 2024-11-14T00:28:08,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:08,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:08,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:08,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:08,766 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:28:08,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:08,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:08,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:28:08,766 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:28:08,768 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:28:08,768 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/A is initiating minor compaction (all files) 2024-11-14T00:28:08,768 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:28:08,768 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/A in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:08,768 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/B is initiating minor compaction (all files) 2024-11-14T00:28:08,768 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/B in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:08,768 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/8eb547e24e8b4e46bc91139f917b5763, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/04982b8567c842679a294094f9913038, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/244312e34c6045a3a144cf1d04f66e56, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/fe4ad4171d2849b08a4d862c6766661a] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=129.9 K 2024-11-14T00:28:08,768 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/b28de671374443acb15e8ab0174e46c1, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/d1c02a080fc24518b7f26e37922a7bfe, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/f7e47a0721f6452b863caf20e3c8d8b0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/4fcd0c8bbccf4707af545ee965a58ae8] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=47.4 K 2024-11-14T00:28:08,768 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:08,768 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/8eb547e24e8b4e46bc91139f917b5763, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/04982b8567c842679a294094f9913038, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/244312e34c6045a3a144cf1d04f66e56, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/fe4ad4171d2849b08a4d862c6766661a] 2024-11-14T00:28:08,769 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting b28de671374443acb15e8ab0174e46c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731544083961 2024-11-14T00:28:08,769 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting d1c02a080fc24518b7f26e37922a7bfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731544084368 2024-11-14T00:28:08,769 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eb547e24e8b4e46bc91139f917b5763, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731544083961 2024-11-14T00:28:08,770 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04982b8567c842679a294094f9913038, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731544084368 2024-11-14T00:28:08,770 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting f7e47a0721f6452b863caf20e3c8d8b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731544085610 2024-11-14T00:28:08,771 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 244312e34c6045a3a144cf1d04f66e56, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731544085610 2024-11-14T00:28:08,771 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fcd0c8bbccf4707af545ee965a58ae8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731544086884 2024-11-14T00:28:08,771 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe4ad4171d2849b08a4d862c6766661a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731544086884 2024-11-14T00:28:08,781 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:08,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T00:28:08,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:08,805 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:08,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T00:28:08,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:08,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:08,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:08,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:08,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:08,811 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#B#compaction#118 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:08,811 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/e830214dccd147738e52180ac6d2b5c9 is 50, key is test_row_0/B:col10/1731544086886/Put/seqid=0 2024-11-14T00:28:08,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T00:28:08,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:08,821 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114052d953e30424c59ae9cde536435c5f3_a2caf5d1c25d111104a7450b3aaca862 store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:08,824 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114052d953e30424c59ae9cde536435c5f3_a2caf5d1c25d111104a7450b3aaca862, store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:08,825 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114052d953e30424c59ae9cde536435c5f3_a2caf5d1c25d111104a7450b3aaca862 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:08,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143249bd84a1b34e4a821395e28c68d2fc_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544088105/Put/seqid=0 2024-11-14T00:28:08,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544148859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544148864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544148864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741968_1144 (size=4469) 2024-11-14T00:28:08,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741967_1143 (size=12493) 2024-11-14T00:28:08,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741969_1145 (size=12304) 2024-11-14T00:28:08,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T00:28:08,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:08,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:08,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:08,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:08,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:08,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544148975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544148986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544148987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T00:28:09,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T00:28:09,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:09,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:09,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:09,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:09,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:09,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544149195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:09,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544149195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:09,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:09,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544149195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:09,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T00:28:09,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544149275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:09,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:09,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:09,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:09,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:09,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544149275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:09,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:09,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:09,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:09,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:09,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:09,301 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#A#compaction#117 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:09,302 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/b345316a8c68446a8baee5b021341efb is 175, key is test_row_0/A:col10/1731544086886/Put/seqid=0 2024-11-14T00:28:09,327 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:09,331 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/e830214dccd147738e52180ac6d2b5c9 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/e830214dccd147738e52180ac6d2b5c9 2024-11-14T00:28:09,334 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143249bd84a1b34e4a821395e28c68d2fc_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143249bd84a1b34e4a821395e28c68d2fc_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:09,336 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/e6547521538046c5a70b991b742ec886, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:09,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/e6547521538046c5a70b991b742ec886 is 175, key is test_row_0/A:col10/1731544088105/Put/seqid=0 2024-11-14T00:28:09,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741970_1146 (size=31447) 2024-11-14T00:28:09,353 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/B of a2caf5d1c25d111104a7450b3aaca862 into e830214dccd147738e52180ac6d2b5c9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:09,353 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:09,354 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/B, priority=12, startTime=1731544088766; duration=0sec 2024-11-14T00:28:09,354 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:09,354 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:B 2024-11-14T00:28:09,354 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:28:09,355 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/b345316a8c68446a8baee5b021341efb as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/b345316a8c68446a8baee5b021341efb 2024-11-14T00:28:09,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741971_1147 (size=31105) 2024-11-14T00:28:09,399 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/e6547521538046c5a70b991b742ec886 2024-11-14T00:28:09,404 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:28:09,404 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/C is initiating minor compaction (all files) 2024-11-14T00:28:09,404 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/C in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:09,404 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/A of a2caf5d1c25d111104a7450b3aaca862 into b345316a8c68446a8baee5b021341efb(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:09,405 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:09,405 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/A, priority=12, startTime=1731544088766; duration=0sec 2024-11-14T00:28:09,405 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/b6272871c2b54c6b8cd78203204905c5, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/0f3a39220fe340dfac55a9c9109bf621, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/e6c59268de854064bdf359fdac5e2493, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/64092ab7132d421ab422eef2f9f5d546] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=47.4 K 2024-11-14T00:28:09,405 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:09,405 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:A 2024-11-14T00:28:09,406 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting b6272871c2b54c6b8cd78203204905c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731544083961 2024-11-14T00:28:09,407 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f3a39220fe340dfac55a9c9109bf621, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731544084368 2024-11-14T00:28:09,408 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting e6c59268de854064bdf359fdac5e2493, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731544085610 2024-11-14T00:28:09,411 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 64092ab7132d421ab422eef2f9f5d546, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731544086884 2024-11-14T00:28:09,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/2e83ad7b7e23496eaf07dc17132fdcaf is 50, key is test_row_0/B:col10/1731544088105/Put/seqid=0 2024-11-14T00:28:09,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T00:28:09,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:09,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:09,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:09,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:09,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:09,467 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#C#compaction#121 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:09,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741972_1148 (size=12151) 2024-11-14T00:28:09,470 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/1350ea79bf7246d28369b579dc1fc01a is 50, key is test_row_0/C:col10/1731544086886/Put/seqid=0 2024-11-14T00:28:09,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/2e83ad7b7e23496eaf07dc17132fdcaf 2024-11-14T00:28:09,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741973_1149 (size=12493) 2024-11-14T00:28:09,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544149506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:09,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544149506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:09,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:09,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:09,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:09,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:09,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:09,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:09,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544149507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:09,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:09,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:09,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:09,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/1a5c2c36e88a4ad588d2030c5dd77e5d is 50, key is test_row_0/C:col10/1731544088105/Put/seqid=0 2024-11-14T00:28:09,520 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/1350ea79bf7246d28369b579dc1fc01a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1350ea79bf7246d28369b579dc1fc01a 2024-11-14T00:28:09,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741974_1150 (size=12151) 2024-11-14T00:28:09,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/1a5c2c36e88a4ad588d2030c5dd77e5d 2024-11-14T00:28:09,537 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/C of a2caf5d1c25d111104a7450b3aaca862 into 1350ea79bf7246d28369b579dc1fc01a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:09,537 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:09,537 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/C, priority=12, startTime=1731544088766; duration=0sec 2024-11-14T00:28:09,537 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:09,538 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:C 2024-11-14T00:28:09,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/e6547521538046c5a70b991b742ec886 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/e6547521538046c5a70b991b742ec886 2024-11-14T00:28:09,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/e6547521538046c5a70b991b742ec886, entries=150, sequenceid=173, filesize=30.4 K 2024-11-14T00:28:09,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/2e83ad7b7e23496eaf07dc17132fdcaf as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/2e83ad7b7e23496eaf07dc17132fdcaf 2024-11-14T00:28:09,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/2e83ad7b7e23496eaf07dc17132fdcaf, entries=150, sequenceid=173, filesize=11.9 K 2024-11-14T00:28:09,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/1a5c2c36e88a4ad588d2030c5dd77e5d as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1a5c2c36e88a4ad588d2030c5dd77e5d 2024-11-14T00:28:09,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1a5c2c36e88a4ad588d2030c5dd77e5d, entries=150, sequenceid=173, filesize=11.9 K 2024-11-14T00:28:09,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for a2caf5d1c25d111104a7450b3aaca862 in 792ms, sequenceid=173, compaction requested=false 2024-11-14T00:28:09,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:09,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T00:28:09,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:09,593 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T00:28:09,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111452102f0d66b64c1eb039335ad6f5901c_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544088843/Put/seqid=0 2024-11-14T00:28:09,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741975_1151 (size=12304) 2024-11-14T00:28:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T00:28:09,967 INFO [master/63baa11b75f3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T00:28:09,967 INFO [master/63baa11b75f3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T00:28:10,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:10,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:10,042 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111452102f0d66b64c1eb039335ad6f5901c_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111452102f0d66b64c1eb039335ad6f5901c_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:10,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/744d26f646a6487e84cfbf68a5f2d5de, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:10,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/744d26f646a6487e84cfbf68a5f2d5de is 175, key is test_row_0/A:col10/1731544088843/Put/seqid=0 2024-11-14T00:28:10,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741976_1152 (size=31105) 2024-11-14T00:28:10,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544150054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:10,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:10,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544150054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:10,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:10,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544150061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:10,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:10,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544150165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544150165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:10,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:10,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544150176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:10,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544150285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:10,291 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_1' in a2caf5d1c25d111104a7450b3aaca862 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:28:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:10,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:10,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544150312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:10,315 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_2' in a2caf5d1c25d111104a7450b3aaca862 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:28:10,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,316 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:10,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544150376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:10,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544150376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:10,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544150385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:10,449 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/744d26f646a6487e84cfbf68a5f2d5de 2024-11-14T00:28:10,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/c0092b083a264801ad7c17dfd2441027 is 50, key is test_row_0/B:col10/1731544088843/Put/seqid=0 2024-11-14T00:28:10,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741977_1153 (size=12151) 2024-11-14T00:28:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T00:28:10,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:10,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544150685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:10,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544150685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:10,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544150705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:10,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:10,924 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/c0092b083a264801ad7c17dfd2441027 2024-11-14T00:28:10,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/6fc660f6dd1646edb7040b0df230c5d1 is 50, key is test_row_0/C:col10/1731544088843/Put/seqid=0 2024-11-14T00:28:10,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741978_1154 (size=12151) 2024-11-14T00:28:11,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544151195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:11,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:11,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544151215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:11,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:11,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544151216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:11,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:11,371 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/6fc660f6dd1646edb7040b0df230c5d1 2024-11-14T00:28:11,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/744d26f646a6487e84cfbf68a5f2d5de as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/744d26f646a6487e84cfbf68a5f2d5de 2024-11-14T00:28:11,389 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/744d26f646a6487e84cfbf68a5f2d5de, entries=150, sequenceid=202, filesize=30.4 K 2024-11-14T00:28:11,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/c0092b083a264801ad7c17dfd2441027 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/c0092b083a264801ad7c17dfd2441027 2024-11-14T00:28:11,401 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/c0092b083a264801ad7c17dfd2441027, entries=150, sequenceid=202, filesize=11.9 K 2024-11-14T00:28:11,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/6fc660f6dd1646edb7040b0df230c5d1 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/6fc660f6dd1646edb7040b0df230c5d1 2024-11-14T00:28:11,412 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/6fc660f6dd1646edb7040b0df230c5d1, entries=150, sequenceid=202, filesize=11.9 K 2024-11-14T00:28:11,413 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for a2caf5d1c25d111104a7450b3aaca862 in 1820ms, sequenceid=202, compaction requested=true 2024-11-14T00:28:11,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:11,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:11,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-14T00:28:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-14T00:28:11,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-14T00:28:11,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9230 sec 2024-11-14T00:28:11,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.9330 sec 2024-11-14T00:28:12,217 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:12,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T00:28:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:12,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114036264b7f72b46ef891305bb5039a4b2_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544090036/Put/seqid=0 2024-11-14T00:28:12,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741979_1155 (size=14794) 2024-11-14T00:28:12,310 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:12,322 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114036264b7f72b46ef891305bb5039a4b2_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114036264b7f72b46ef891305bb5039a4b2_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:12,324 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/8935f4cdcdfa48088203019098f6fa2e, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:12,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/8935f4cdcdfa48088203019098f6fa2e is 175, key is test_row_0/A:col10/1731544090036/Put/seqid=0 2024-11-14T00:28:12,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741980_1156 (size=39749) 2024-11-14T00:28:12,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544152346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:12,363 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_2' in a2caf5d1c25d111104a7450b3aaca862 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:28:12,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:12,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544152353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:12,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544152353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:12,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:12,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544152356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544152353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:12,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:12,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:12,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544152481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:12,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544152481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:12,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544152482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:12,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544152483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:12,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T00:28:12,664 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:12,665 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-14T00:28:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T00:28:12,668 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:12,671 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:12,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:12,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544152695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:12,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544152695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:12,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544152695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:12,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544152696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:12,743 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/8935f4cdcdfa48088203019098f6fa2e 2024-11-14T00:28:12,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/7f6041dc63904605b478476f7540cd22 is 50, key is test_row_0/B:col10/1731544090036/Put/seqid=0 2024-11-14T00:28:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T00:28:12,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741981_1157 (size=12151) 2024-11-14T00:28:12,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T00:28:12,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:12,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:12,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:12,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:12,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:12,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:12,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T00:28:12,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T00:28:12,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:12,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:12,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:12,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:12,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:12,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:13,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544153006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544153006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:13,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544153007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:13,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544153007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:13,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:13,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T00:28:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:13,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:13,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:13,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:13,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/7f6041dc63904605b478476f7540cd22 2024-11-14T00:28:13,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/bddf8d214eb140b29cb824de857b397e is 50, key is test_row_0/C:col10/1731544090036/Put/seqid=0 2024-11-14T00:28:13,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741982_1158 (size=12151) 2024-11-14T00:28:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T00:28:13,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/bddf8d214eb140b29cb824de857b397e 2024-11-14T00:28:13,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T00:28:13,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:13,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:13,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:13,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:13,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:13,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/8935f4cdcdfa48088203019098f6fa2e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/8935f4cdcdfa48088203019098f6fa2e 2024-11-14T00:28:13,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/8935f4cdcdfa48088203019098f6fa2e, entries=200, sequenceid=213, filesize=38.8 K 2024-11-14T00:28:13,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/7f6041dc63904605b478476f7540cd22 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/7f6041dc63904605b478476f7540cd22 2024-11-14T00:28:13,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/7f6041dc63904605b478476f7540cd22, entries=150, sequenceid=213, filesize=11.9 K 2024-11-14T00:28:13,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/bddf8d214eb140b29cb824de857b397e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/bddf8d214eb140b29cb824de857b397e 2024-11-14T00:28:13,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/bddf8d214eb140b29cb824de857b397e, entries=150, sequenceid=213, filesize=11.9 K 2024-11-14T00:28:13,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a2caf5d1c25d111104a7450b3aaca862 in 1138ms, sequenceid=213, compaction requested=true 2024-11-14T00:28:13,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:13,356 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:28:13,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:13,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:13,359 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133406 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:28:13,359 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/A is initiating minor compaction (all files) 2024-11-14T00:28:13,359 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/A in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:13,359 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/b345316a8c68446a8baee5b021341efb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/e6547521538046c5a70b991b742ec886, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/744d26f646a6487e84cfbf68a5f2d5de, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/8935f4cdcdfa48088203019098f6fa2e] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=130.3 K 2024-11-14T00:28:13,359 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:13,359 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/b345316a8c68446a8baee5b021341efb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/e6547521538046c5a70b991b742ec886, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/744d26f646a6487e84cfbf68a5f2d5de, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/8935f4cdcdfa48088203019098f6fa2e] 2024-11-14T00:28:13,360 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting b345316a8c68446a8baee5b021341efb, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731544086884 2024-11-14T00:28:13,361 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6547521538046c5a70b991b742ec886, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731544088081 2024-11-14T00:28:13,362 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 744d26f646a6487e84cfbf68a5f2d5de, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731544088843 2024-11-14T00:28:13,363 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:28:13,364 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8935f4cdcdfa48088203019098f6fa2e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731544090035 2024-11-14T00:28:13,368 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:28:13,368 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/B is initiating minor compaction (all files) 2024-11-14T00:28:13,368 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/B in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:13,369 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/e830214dccd147738e52180ac6d2b5c9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/2e83ad7b7e23496eaf07dc17132fdcaf, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/c0092b083a264801ad7c17dfd2441027, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/7f6041dc63904605b478476f7540cd22] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=47.8 K 2024-11-14T00:28:13,370 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting e830214dccd147738e52180ac6d2b5c9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731544086884 2024-11-14T00:28:13,371 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e83ad7b7e23496eaf07dc17132fdcaf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731544088081 2024-11-14T00:28:13,371 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c0092b083a264801ad7c17dfd2441027, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731544088843 2024-11-14T00:28:13,371 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f6041dc63904605b478476f7540cd22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731544090036 2024-11-14T00:28:13,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:13,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:13,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:13,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:13,386 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:13,403 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#B#compaction#130 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:13,404 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114cf40e0fc1ece42d0b03b4cc6b29a92f2_a2caf5d1c25d111104a7450b3aaca862 store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:13,404 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/d92a3193229f4d8981b66a6b47ce13fb is 50, key is test_row_0/B:col10/1731544090036/Put/seqid=0 2024-11-14T00:28:13,408 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114cf40e0fc1ece42d0b03b4cc6b29a92f2_a2caf5d1c25d111104a7450b3aaca862, store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:13,410 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cf40e0fc1ece42d0b03b4cc6b29a92f2_a2caf5d1c25d111104a7450b3aaca862 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:13,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T00:28:13,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:13,464 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T00:28:13,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741983_1159 (size=12629) 2024-11-14T00:28:13,483 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/d92a3193229f4d8981b66a6b47ce13fb as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/d92a3193229f4d8981b66a6b47ce13fb 2024-11-14T00:28:13,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741984_1160 (size=4469) 2024-11-14T00:28:13,491 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#A#compaction#129 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:13,492 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/629d89b91619420ab1f8fb6b448f02f6 is 175, key is test_row_0/A:col10/1731544090036/Put/seqid=0 2024-11-14T00:28:13,493 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/B of a2caf5d1c25d111104a7450b3aaca862 into d92a3193229f4d8981b66a6b47ce13fb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:13,493 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:13,493 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/B, priority=12, startTime=1731544093363; duration=0sec 2024-11-14T00:28:13,493 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:13,493 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:B 2024-11-14T00:28:13,493 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:28:13,497 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:28:13,498 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/C is initiating minor compaction (all files) 2024-11-14T00:28:13,498 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/C in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:13,498 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1350ea79bf7246d28369b579dc1fc01a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1a5c2c36e88a4ad588d2030c5dd77e5d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/6fc660f6dd1646edb7040b0df230c5d1, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/bddf8d214eb140b29cb824de857b397e] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=47.8 K 2024-11-14T00:28:13,499 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 1350ea79bf7246d28369b579dc1fc01a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731544086884 2024-11-14T00:28:13,499 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a5c2c36e88a4ad588d2030c5dd77e5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731544088081 2024-11-14T00:28:13,500 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fc660f6dd1646edb7040b0df230c5d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731544088843 2024-11-14T00:28:13,503 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting bddf8d214eb140b29cb824de857b397e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731544090036 2024-11-14T00:28:13,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c78bb357019e4c7c93b68426d758df89_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544092331/Put/seqid=0 2024-11-14T00:28:13,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741985_1161 (size=31583) 2024-11-14T00:28:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:13,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:13,553 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#C#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:13,554 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/3824bc883edf470c973c71d7aaf94b51 is 50, key is test_row_0/C:col10/1731544090036/Put/seqid=0 2024-11-14T00:28:13,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544153566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:13,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741986_1162 (size=12304) 2024-11-14T00:28:13,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:13,585 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c78bb357019e4c7c93b68426d758df89_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c78bb357019e4c7c93b68426d758df89_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:13,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/dc357625bc3d493fa09317b00d46fda4, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:13,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/dc357625bc3d493fa09317b00d46fda4 is 175, key is test_row_0/A:col10/1731544092331/Put/seqid=0 2024-11-14T00:28:13,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544153576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:13,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544153572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:13,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544153576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:13,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741987_1163 (size=12629) 2024-11-14T00:28:13,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741988_1164 (size=31105) 2024-11-14T00:28:13,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544153688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:13,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544153696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:13,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544153696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:13,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544153712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T00:28:13,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:13,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544153895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:13,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544153906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:13,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544153906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:13,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:13,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544153925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:13,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:13,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:13,959 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/629d89b91619420ab1f8fb6b448f02f6 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/629d89b91619420ab1f8fb6b448f02f6 2024-11-14T00:28:13,970 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/A of a2caf5d1c25d111104a7450b3aaca862 into 629d89b91619420ab1f8fb6b448f02f6(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:13,970 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:13,970 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/A, priority=12, startTime=1731544093356; duration=0sec 2024-11-14T00:28:13,970 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:13,970 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:A 2024-11-14T00:28:14,032 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/3824bc883edf470c973c71d7aaf94b51 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/3824bc883edf470c973c71d7aaf94b51 2024-11-14T00:28:14,050 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/C of a2caf5d1c25d111104a7450b3aaca862 into 3824bc883edf470c973c71d7aaf94b51(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:14,050 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:14,050 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/C, priority=12, startTime=1731544093374; duration=0sec 2024-11-14T00:28:14,050 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:14,050 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:C 2024-11-14T00:28:14,058 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/dc357625bc3d493fa09317b00d46fda4 2024-11-14T00:28:14,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/0a593ac3387f4ef48a2d9b49998f9dc0 is 50, key is test_row_0/B:col10/1731544092331/Put/seqid=0 2024-11-14T00:28:14,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741989_1165 (size=12151) 2024-11-14T00:28:14,123 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/0a593ac3387f4ef48a2d9b49998f9dc0 2024-11-14T00:28:14,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/ac9120b2bd3d449bb7074fb693e73c63 is 50, key is test_row_0/C:col10/1731544092331/Put/seqid=0 2024-11-14T00:28:14,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741990_1166 (size=12151) 2024-11-14T00:28:14,201 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/ac9120b2bd3d449bb7074fb693e73c63 2024-11-14T00:28:14,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/dc357625bc3d493fa09317b00d46fda4 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/dc357625bc3d493fa09317b00d46fda4 2024-11-14T00:28:14,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:14,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544154205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:14,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:14,217 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/dc357625bc3d493fa09317b00d46fda4, entries=150, sequenceid=238, filesize=30.4 K 2024-11-14T00:28:14,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/0a593ac3387f4ef48a2d9b49998f9dc0 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/0a593ac3387f4ef48a2d9b49998f9dc0 2024-11-14T00:28:14,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544154216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:14,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544154216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:14,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:14,228 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/0a593ac3387f4ef48a2d9b49998f9dc0, entries=150, sequenceid=238, filesize=11.9 K 2024-11-14T00:28:14,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/ac9120b2bd3d449bb7074fb693e73c63 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/ac9120b2bd3d449bb7074fb693e73c63 2024-11-14T00:28:14,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544154235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:14,238 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/ac9120b2bd3d449bb7074fb693e73c63, entries=150, sequenceid=238, filesize=11.9 K 2024-11-14T00:28:14,242 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a2caf5d1c25d111104a7450b3aaca862 in 775ms, sequenceid=238, compaction requested=false 2024-11-14T00:28:14,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:14,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:14,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-14T00:28:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-14T00:28:14,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-14T00:28:14,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5760 sec 2024-11-14T00:28:14,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.5900 sec 2024-11-14T00:28:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:14,719 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:14,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T00:28:14,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111493660e8b3ff64e7584f62401e40f496a_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544093571/Put/seqid=0 2024-11-14T00:28:14,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741991_1167 (size=12304) 2024-11-14T00:28:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T00:28:14,816 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:14,817 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-14T00:28:14,819 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:14,821 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:14,821 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T00:28:14,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544154810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:14,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544154812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544154815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:14,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544154820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:14,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544154925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:14,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T00:28:14,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:14,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544154935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544154935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:14,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544154935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:14,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T00:28:14,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:14,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:14,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:14,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:14,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:15,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T00:28:15,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:15,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:15,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:15,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:15,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T00:28:15,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544155145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:15,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544155145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:15,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:15,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544155146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:15,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:15,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544155156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:15,199 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:15,206 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111493660e8b3ff64e7584f62401e40f496a_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111493660e8b3ff64e7584f62401e40f496a_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:15,208 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/aff8b62f23d140749e267a9d39cbe183, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:15,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/aff8b62f23d140749e267a9d39cbe183 is 175, key is test_row_0/A:col10/1731544093571/Put/seqid=0 2024-11-14T00:28:15,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741992_1168 (size=31105) 2024-11-14T00:28:15,261 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/aff8b62f23d140749e267a9d39cbe183 2024-11-14T00:28:15,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/66188815ebc54eee84dca59e7aa31a3f is 50, key is test_row_0/B:col10/1731544093571/Put/seqid=0 2024-11-14T00:28:15,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T00:28:15,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:15,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:15,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:15,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:15,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:15,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741993_1169 (size=12151) 2024-11-14T00:28:15,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/66188815ebc54eee84dca59e7aa31a3f 2024-11-14T00:28:15,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/1c76fc77f09743749f680af57d4aa088 is 50, key is test_row_0/C:col10/1731544093571/Put/seqid=0 2024-11-14T00:28:15,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741994_1170 (size=12151) 2024-11-14T00:28:15,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T00:28:15,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:15,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:15,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:15,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:15,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T00:28:15,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544155466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:15,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544155466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:15,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:15,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544155471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:15,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:15,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:15,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:15,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544155472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:15,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T00:28:15,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:15,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:15,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:15,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:15,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:15,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T00:28:15,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:15,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:15,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:15,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:15,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:15,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/1c76fc77f09743749f680af57d4aa088 2024-11-14T00:28:15,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/aff8b62f23d140749e267a9d39cbe183 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/aff8b62f23d140749e267a9d39cbe183 2024-11-14T00:28:15,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/aff8b62f23d140749e267a9d39cbe183, entries=150, sequenceid=253, filesize=30.4 K 2024-11-14T00:28:15,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/66188815ebc54eee84dca59e7aa31a3f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/66188815ebc54eee84dca59e7aa31a3f 2024-11-14T00:28:15,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/66188815ebc54eee84dca59e7aa31a3f, entries=150, sequenceid=253, filesize=11.9 K 2024-11-14T00:28:15,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/1c76fc77f09743749f680af57d4aa088 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1c76fc77f09743749f680af57d4aa088 2024-11-14T00:28:15,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1c76fc77f09743749f680af57d4aa088, entries=150, sequenceid=253, filesize=11.9 K 2024-11-14T00:28:15,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for a2caf5d1c25d111104a7450b3aaca862 in 1114ms, sequenceid=253, compaction requested=true 2024-11-14T00:28:15,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:15,833 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:15,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:15,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:15,834 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:15,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:15,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:15,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:15,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:15,836 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93793 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:15,836 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/A is initiating minor compaction (all files) 2024-11-14T00:28:15,836 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/A in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:15,836 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/629d89b91619420ab1f8fb6b448f02f6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/dc357625bc3d493fa09317b00d46fda4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/aff8b62f23d140749e267a9d39cbe183] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=91.6 K 2024-11-14T00:28:15,836 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:15,836 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/629d89b91619420ab1f8fb6b448f02f6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/dc357625bc3d493fa09317b00d46fda4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/aff8b62f23d140749e267a9d39cbe183] 2024-11-14T00:28:15,837 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:15,837 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 629d89b91619420ab1f8fb6b448f02f6, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731544090036 2024-11-14T00:28:15,837 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/B is initiating minor compaction (all files) 2024-11-14T00:28:15,837 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/B in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:15,837 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/d92a3193229f4d8981b66a6b47ce13fb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/0a593ac3387f4ef48a2d9b49998f9dc0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/66188815ebc54eee84dca59e7aa31a3f] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=36.1 K 2024-11-14T00:28:15,838 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc357625bc3d493fa09317b00d46fda4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731544092331 2024-11-14T00:28:15,838 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting d92a3193229f4d8981b66a6b47ce13fb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731544090036 2024-11-14T00:28:15,838 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting aff8b62f23d140749e267a9d39cbe183, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731544093568 2024-11-14T00:28:15,839 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a593ac3387f4ef48a2d9b49998f9dc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731544092331 2024-11-14T00:28:15,840 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 66188815ebc54eee84dca59e7aa31a3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731544093568 2024-11-14T00:28:15,867 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:15,870 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#B#compaction#139 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:15,871 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/8f1b7651bc4a4740a2044306fb675017 is 50, key is test_row_0/B:col10/1731544093571/Put/seqid=0 2024-11-14T00:28:15,881 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411148bb3e700eef24d089a83f096695cafce_a2caf5d1c25d111104a7450b3aaca862 store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:15,884 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411148bb3e700eef24d089a83f096695cafce_a2caf5d1c25d111104a7450b3aaca862, store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:15,884 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148bb3e700eef24d089a83f096695cafce_a2caf5d1c25d111104a7450b3aaca862 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:15,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T00:28:15,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:15,907 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T00:28:15,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741995_1171 (size=12731) 2024-11-14T00:28:15,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741996_1172 (size=4469) 2024-11-14T00:28:15,957 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#A#compaction#138 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:15,959 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/478a0fb5ad734ef8b17868c16bc66295 is 175, key is test_row_0/A:col10/1731544093571/Put/seqid=0 2024-11-14T00:28:15,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c1b0c9aa69494593bfff8b16f78c510e_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544094813/Put/seqid=0 2024-11-14T00:28:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T00:28:15,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:16,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741997_1173 (size=31685) 2024-11-14T00:28:16,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741998_1174 (size=12454) 2024-11-14T00:28:16,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544156006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544156006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544156018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544156019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544156125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544156135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544156135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544156135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,333 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/8f1b7651bc4a4740a2044306fb675017 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/8f1b7651bc4a4740a2044306fb675017 2024-11-14T00:28:16,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544156335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,342 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/B of a2caf5d1c25d111104a7450b3aaca862 into 8f1b7651bc4a4740a2044306fb675017(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:16,342 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:16,342 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/B, priority=13, startTime=1731544095834; duration=0sec 2024-11-14T00:28:16,342 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:16,342 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:B 2024-11-14T00:28:16,342 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:16,344 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:16,344 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/C is initiating minor compaction (all files) 2024-11-14T00:28:16,344 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/C in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:16,344 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/3824bc883edf470c973c71d7aaf94b51, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/ac9120b2bd3d449bb7074fb693e73c63, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1c76fc77f09743749f680af57d4aa088] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=36.1 K 2024-11-14T00:28:16,345 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 3824bc883edf470c973c71d7aaf94b51, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731544090036 2024-11-14T00:28:16,347 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting ac9120b2bd3d449bb7074fb693e73c63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731544092331 2024-11-14T00:28:16,348 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c76fc77f09743749f680af57d4aa088, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731544093568 2024-11-14T00:28:16,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544156356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544156356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544156357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,372 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:16,372 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/a2eb0381d5324376be7659beb04c25d5 is 50, key is test_row_0/C:col10/1731544093571/Put/seqid=0 2024-11-14T00:28:16,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35382 deadline: 1731544156385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,388 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_2' in a2caf5d1c25d111104a7450b3aaca862 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:28:16,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741999_1175 (size=12731) 2024-11-14T00:28:16,405 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/a2eb0381d5324376be7659beb04c25d5 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/a2eb0381d5324376be7659beb04c25d5 2024-11-14T00:28:16,416 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/478a0fb5ad734ef8b17868c16bc66295 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/478a0fb5ad734ef8b17868c16bc66295 2024-11-14T00:28:16,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:16,422 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/C of a2caf5d1c25d111104a7450b3aaca862 into a2eb0381d5324376be7659beb04c25d5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:16,422 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:16,422 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/C, priority=13, startTime=1731544095834; duration=0sec 2024-11-14T00:28:16,422 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:16,422 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:C 2024-11-14T00:28:16,425 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/A of a2caf5d1c25d111104a7450b3aaca862 into 478a0fb5ad734ef8b17868c16bc66295(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:16,425 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:16,425 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/A, priority=13, startTime=1731544095833; duration=0sec 2024-11-14T00:28:16,425 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:16,426 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:A 2024-11-14T00:28:16,428 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c1b0c9aa69494593bfff8b16f78c510e_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c1b0c9aa69494593bfff8b16f78c510e_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:16,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/0f813ff3e08847fd9a2cd34bab60bec7, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:16,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/0f813ff3e08847fd9a2cd34bab60bec7 is 175, key is test_row_0/A:col10/1731544094813/Put/seqid=0 2024-11-14T00:28:16,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742000_1176 (size=31255) 2024-11-14T00:28:16,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544156646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544156665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:16,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544156665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544156665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:16,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:16,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:16,861 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/0f813ff3e08847fd9a2cd34bab60bec7 2024-11-14T00:28:16,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/bf2d95c322ef463c812e92672bcb1322 is 50, key is test_row_0/B:col10/1731544094813/Put/seqid=0 2024-11-14T00:28:16,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742001_1177 (size=12301) 2024-11-14T00:28:16,914 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/bf2d95c322ef463c812e92672bcb1322 2024-11-14T00:28:16,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/c89c3e4c181d4a6d847d43ae93b14c0a is 50, key is test_row_0/C:col10/1731544094813/Put/seqid=0 2024-11-14T00:28:16,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742002_1178 (size=12301) 2024-11-14T00:28:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T00:28:17,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544157156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:17,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544157175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:17,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:17,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:17,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:17,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544157185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:17,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:17,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544157186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:17,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:17,357 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/c89c3e4c181d4a6d847d43ae93b14c0a 2024-11-14T00:28:17,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/0f813ff3e08847fd9a2cd34bab60bec7 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/0f813ff3e08847fd9a2cd34bab60bec7 2024-11-14T00:28:17,380 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/0f813ff3e08847fd9a2cd34bab60bec7, entries=150, sequenceid=278, filesize=30.5 K 2024-11-14T00:28:17,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/bf2d95c322ef463c812e92672bcb1322 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/bf2d95c322ef463c812e92672bcb1322 2024-11-14T00:28:17,399 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/bf2d95c322ef463c812e92672bcb1322, entries=150, sequenceid=278, filesize=12.0 K 2024-11-14T00:28:17,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/c89c3e4c181d4a6d847d43ae93b14c0a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/c89c3e4c181d4a6d847d43ae93b14c0a 2024-11-14T00:28:17,415 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/c89c3e4c181d4a6d847d43ae93b14c0a, entries=150, sequenceid=278, filesize=12.0 K 2024-11-14T00:28:17,417 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a2caf5d1c25d111104a7450b3aaca862 in 1510ms, sequenceid=278, compaction requested=false 2024-11-14T00:28:17,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:17,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:17,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-14T00:28:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-14T00:28:17,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-14T00:28:17,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5980 sec 2024-11-14T00:28:17,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.6080 sec 2024-11-14T00:28:18,178 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:18,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T00:28:18,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:18,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cf8e634961924299a081187a52d760bf_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544098176/Put/seqid=0 2024-11-14T00:28:18,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544158256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:18,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:18,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544158258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:18,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544158258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:18,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:18,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544158261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:18,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:18,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742003_1179 (size=14994) 2024-11-14T00:28:18,271 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:18,280 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cf8e634961924299a081187a52d760bf_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cf8e634961924299a081187a52d760bf_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:18,282 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/a62efab2860d4dcca7c187c2114e86ba, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:18,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/a62efab2860d4dcca7c187c2114e86ba is 175, key is test_row_0/A:col10/1731544098176/Put/seqid=0 2024-11-14T00:28:18,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742004_1180 (size=39949) 2024-11-14T00:28:18,326 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/a62efab2860d4dcca7c187c2114e86ba 2024-11-14T00:28:18,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/9768bf75838940a0b90686f4e4014402 is 50, key is test_row_0/B:col10/1731544098176/Put/seqid=0 2024-11-14T00:28:18,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742005_1181 (size=12301) 2024-11-14T00:28:18,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544158376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:18,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544158376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544158376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:18,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544158376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:18,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/9768bf75838940a0b90686f4e4014402 2024-11-14T00:28:18,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/6bdada454cd043d1809fe6ae5680447b is 50, key is test_row_0/C:col10/1731544098176/Put/seqid=0 2024-11-14T00:28:18,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742006_1182 (size=12301) 2024-11-14T00:28:18,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544158585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:18,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544158585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:18,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544158585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:18,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544158586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:18,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/6bdada454cd043d1809fe6ae5680447b 2024-11-14T00:28:18,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/a62efab2860d4dcca7c187c2114e86ba as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/a62efab2860d4dcca7c187c2114e86ba 2024-11-14T00:28:18,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544158895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:18,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:18,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544158895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:18,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:18,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:18,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544158895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:18,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544158895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:18,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/a62efab2860d4dcca7c187c2114e86ba, entries=200, sequenceid=293, filesize=39.0 K 2024-11-14T00:28:18,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/9768bf75838940a0b90686f4e4014402 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/9768bf75838940a0b90686f4e4014402 2024-11-14T00:28:18,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/9768bf75838940a0b90686f4e4014402, entries=150, sequenceid=293, filesize=12.0 K 2024-11-14T00:28:18,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/6bdada454cd043d1809fe6ae5680447b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/6bdada454cd043d1809fe6ae5680447b 2024-11-14T00:28:18,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/6bdada454cd043d1809fe6ae5680447b, entries=150, sequenceid=293, filesize=12.0 K 2024-11-14T00:28:18,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a2caf5d1c25d111104a7450b3aaca862 in 759ms, sequenceid=293, compaction requested=true 2024-11-14T00:28:18,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:18,937 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:18,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:18,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:18,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:18,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:18,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:18,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:28:18,941 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:18,941 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:18,941 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/A is initiating minor compaction (all files) 2024-11-14T00:28:18,941 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/A in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:18,941 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/478a0fb5ad734ef8b17868c16bc66295, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/0f813ff3e08847fd9a2cd34bab60bec7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/a62efab2860d4dcca7c187c2114e86ba] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=100.5 K 2024-11-14T00:28:18,941 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:18,941 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/478a0fb5ad734ef8b17868c16bc66295, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/0f813ff3e08847fd9a2cd34bab60bec7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/a62efab2860d4dcca7c187c2114e86ba] 2024-11-14T00:28:18,942 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 478a0fb5ad734ef8b17868c16bc66295, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731544093568 2024-11-14T00:28:18,943 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f813ff3e08847fd9a2cd34bab60bec7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731544094808 2024-11-14T00:28:18,943 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting a62efab2860d4dcca7c187c2114e86ba, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731544096002 2024-11-14T00:28:18,944 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:18,944 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/B is initiating minor compaction (all files) 2024-11-14T00:28:18,944 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/B in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:18,945 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/8f1b7651bc4a4740a2044306fb675017, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/bf2d95c322ef463c812e92672bcb1322, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/9768bf75838940a0b90686f4e4014402] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=36.5 K 2024-11-14T00:28:18,946 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f1b7651bc4a4740a2044306fb675017, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731544093568 2024-11-14T00:28:18,947 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting bf2d95c322ef463c812e92672bcb1322, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731544094808 2024-11-14T00:28:18,948 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 9768bf75838940a0b90686f4e4014402, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731544096002 2024-11-14T00:28:18,965 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:18,989 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#B#compaction#148 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:18,990 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/f630eef01d294304bf9922feb7698807 is 50, key is test_row_0/B:col10/1731544098176/Put/seqid=0 2024-11-14T00:28:18,992 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411140fff22a5845e4e74b1c2dc67c5b9515a_a2caf5d1c25d111104a7450b3aaca862 store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T00:28:18,995 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:18,996 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-14T00:28:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T00:28:19,002 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:19,004 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:19,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:19,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742007_1183 (size=12983) 2024-11-14T00:28:19,005 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411140fff22a5845e4e74b1c2dc67c5b9515a_a2caf5d1c25d111104a7450b3aaca862, store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:19,006 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140fff22a5845e4e74b1c2dc67c5b9515a_a2caf5d1c25d111104a7450b3aaca862 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:19,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742008_1184 (size=4469) 2024-11-14T00:28:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T00:28:19,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T00:28:19,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:19,160 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T00:28:19,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145725aabe6418405799851fe383e19ec1_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544098258/Put/seqid=0 2024-11-14T00:28:19,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742009_1185 (size=12454) 2024-11-14T00:28:19,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:19,203 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145725aabe6418405799851fe383e19ec1_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145725aabe6418405799851fe383e19ec1_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:19,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/ec0966de5c8c4a64878d3b6a666dafc2, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:19,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/ec0966de5c8c4a64878d3b6a666dafc2 is 175, key is test_row_0/A:col10/1731544098258/Put/seqid=0 2024-11-14T00:28:19,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742010_1186 (size=31255) 2024-11-14T00:28:19,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T00:28:19,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:19,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. as already flushing 2024-11-14T00:28:19,420 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/f630eef01d294304bf9922feb7698807 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/f630eef01d294304bf9922feb7698807 2024-11-14T00:28:19,430 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/B of a2caf5d1c25d111104a7450b3aaca862 into f630eef01d294304bf9922feb7698807(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:19,430 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:19,430 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/B, priority=13, startTime=1731544098940; duration=0sec 2024-11-14T00:28:19,430 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:19,430 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:B 2024-11-14T00:28:19,430 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:19,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:19,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544159427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:19,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:19,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544159429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:19,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,433 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:19,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:19,433 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/C is initiating minor compaction (all files) 2024-11-14T00:28:19,433 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/C in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:19,434 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/a2eb0381d5324376be7659beb04c25d5, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/c89c3e4c181d4a6d847d43ae93b14c0a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/6bdada454cd043d1809fe6ae5680447b] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=36.5 K 2024-11-14T00:28:19,434 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting a2eb0381d5324376be7659beb04c25d5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731544093568 2024-11-14T00:28:19,434 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c89c3e4c181d4a6d847d43ae93b14c0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731544094808 2024-11-14T00:28:19,435 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bdada454cd043d1809fe6ae5680447b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731544096002 2024-11-14T00:28:19,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:19,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544159431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544159433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:19,460 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#C#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:19,461 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/822b75dbd53849d38a868e530e73cc3b is 50, key is test_row_0/C:col10/1731544098176/Put/seqid=0 2024-11-14T00:28:19,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742011_1187 (size=12983) 2024-11-14T00:28:19,465 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#A#compaction#147 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:19,465 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/fc03c55bbd914ba1bdf6cc052d901b29 is 175, key is test_row_0/A:col10/1731544098176/Put/seqid=0 2024-11-14T00:28:19,475 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/822b75dbd53849d38a868e530e73cc3b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/822b75dbd53849d38a868e530e73cc3b 2024-11-14T00:28:19,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742012_1188 (size=31937) 2024-11-14T00:28:19,482 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/C of a2caf5d1c25d111104a7450b3aaca862 into 822b75dbd53849d38a868e530e73cc3b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:19,482 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:19,482 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/C, priority=13, startTime=1731544098940; duration=0sec 2024-11-14T00:28:19,482 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:19,482 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:C 2024-11-14T00:28:19,495 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/fc03c55bbd914ba1bdf6cc052d901b29 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/fc03c55bbd914ba1bdf6cc052d901b29 2024-11-14T00:28:19,507 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/A of a2caf5d1c25d111104a7450b3aaca862 into fc03c55bbd914ba1bdf6cc052d901b29(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:19,507 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:19,507 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/A, priority=13, startTime=1731544098937; duration=0sec 2024-11-14T00:28:19,507 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:19,507 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:A 2024-11-14T00:28:19,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544159535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:19,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544159545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544159545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544159545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T00:28:19,635 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/ec0966de5c8c4a64878d3b6a666dafc2 2024-11-14T00:28:19,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/7929338c69c74a1ebb0abb14632ca23b is 50, key is test_row_0/B:col10/1731544098258/Put/seqid=0 2024-11-14T00:28:19,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742013_1189 (size=12301) 2024-11-14T00:28:19,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544159745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:19,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544159755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544159755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:19,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544159755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:19,964 INFO [Thread-550 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-550. 2024-11-14T00:28:19,964 DEBUG [Thread-550 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:19,964 DEBUG [Thread-550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:19,964 DEBUG [Thread-550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:19,964 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:19,967 INFO [Thread-552 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-552. 2024-11-14T00:28:19,967 DEBUG [Thread-552 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:19,967 DEBUG [Thread-552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:19,967 DEBUG [Thread-552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:19,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:19,968 INFO [Thread-548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-548. 2024-11-14T00:28:19,968 DEBUG [Thread-548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:19,968 DEBUG [Thread-548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:19,968 DEBUG [Thread-548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:19,968 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:19,970 INFO [Thread-554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-554. 2024-11-14T00:28:19,970 DEBUG [Thread-554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:19,970 DEBUG [Thread-554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:19,970 DEBUG [Thread-554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:19,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:20,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35462 deadline: 1731544160055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:20,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35404 deadline: 1731544160066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:20,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35436 deadline: 1731544160066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:20,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35420 deadline: 1731544160066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:20,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:20,075 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/7929338c69c74a1ebb0abb14632ca23b 2024-11-14T00:28:20,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/ce040e9f82c74b83824f99a12207cb50 is 50, key is test_row_0/C:col10/1731544098258/Put/seqid=0 2024-11-14T00:28:20,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742014_1190 (size=12301) 2024-11-14T00:28:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T00:28:20,508 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/ce040e9f82c74b83824f99a12207cb50 2024-11-14T00:28:20,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/ec0966de5c8c4a64878d3b6a666dafc2 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/ec0966de5c8c4a64878d3b6a666dafc2 2024-11-14T00:28:20,523 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/ec0966de5c8c4a64878d3b6a666dafc2, entries=150, sequenceid=317, filesize=30.5 K 2024-11-14T00:28:20,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/7929338c69c74a1ebb0abb14632ca23b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/7929338c69c74a1ebb0abb14632ca23b 2024-11-14T00:28:20,533 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/7929338c69c74a1ebb0abb14632ca23b, entries=150, sequenceid=317, filesize=12.0 K 2024-11-14T00:28:20,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/ce040e9f82c74b83824f99a12207cb50 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/ce040e9f82c74b83824f99a12207cb50 2024-11-14T00:28:20,540 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/ce040e9f82c74b83824f99a12207cb50, entries=150, sequenceid=317, filesize=12.0 K 2024-11-14T00:28:20,541 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for a2caf5d1c25d111104a7450b3aaca862 in 1381ms, sequenceid=317, compaction requested=false 2024-11-14T00:28:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:20,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-14T00:28:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-14T00:28:20,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-14T00:28:20,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5390 sec 2024-11-14T00:28:20,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.5510 sec 2024-11-14T00:28:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:20,566 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:20,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T00:28:20,567 INFO [Thread-545 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-545. 2024-11-14T00:28:20,567 DEBUG [Thread-545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:20,567 DEBUG [Thread-545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:20,567 DEBUG [Thread-545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:20,567 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:20,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114daacf1709d4e441c81a4dc65b706c622_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_0/A:col10/1731544100565/Put/seqid=0 2024-11-14T00:28:20,577 INFO [Thread-539 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-539. 2024-11-14T00:28:20,577 INFO [Thread-543 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-543. 2024-11-14T00:28:20,577 INFO [Thread-537 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-537. 2024-11-14T00:28:20,577 DEBUG [Thread-543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:20,577 DEBUG [Thread-539 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:20,578 DEBUG [Thread-537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:20,578 DEBUG [Thread-539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:20,578 DEBUG [Thread-543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:20,578 DEBUG [Thread-537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:20,578 DEBUG [Thread-539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:20,578 DEBUG [Thread-543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:20,578 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:20,578 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:20,578 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:20,578 DEBUG [Thread-537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:20,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742015_1191 (size=12454) 2024-11-14T00:28:20,590 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:20,595 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114daacf1709d4e441c81a4dc65b706c622_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114daacf1709d4e441c81a4dc65b706c622_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:20,596 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/5d22b70aaba041678543085e7aa19bd5, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:20,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/5d22b70aaba041678543085e7aa19bd5 is 175, key is test_row_0/A:col10/1731544100565/Put/seqid=0 2024-11-14T00:28:20,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742016_1192 (size=31255) 2024-11-14T00:28:21,004 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/5d22b70aaba041678543085e7aa19bd5 2024-11-14T00:28:21,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/9dc25cc6f2ef424cb6453dfdaf5c989e is 50, key is test_row_0/B:col10/1731544100565/Put/seqid=0 2024-11-14T00:28:21,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742017_1193 (size=12301) 2024-11-14T00:28:21,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/9dc25cc6f2ef424cb6453dfdaf5c989e 2024-11-14T00:28:21,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/2c44e59652404d32aa76f3da28de9330 is 50, key is test_row_0/C:col10/1731544100565/Put/seqid=0 2024-11-14T00:28:21,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742018_1194 (size=12301) 2024-11-14T00:28:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T00:28:21,147 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:21,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/2c44e59652404d32aa76f3da28de9330 2024-11-14T00:28:21,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/5d22b70aaba041678543085e7aa19bd5 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/5d22b70aaba041678543085e7aa19bd5 2024-11-14T00:28:21,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/5d22b70aaba041678543085e7aa19bd5, entries=150, sequenceid=334, filesize=30.5 K 2024-11-14T00:28:21,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/9dc25cc6f2ef424cb6453dfdaf5c989e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/9dc25cc6f2ef424cb6453dfdaf5c989e 2024-11-14T00:28:21,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/9dc25cc6f2ef424cb6453dfdaf5c989e, entries=150, sequenceid=334, filesize=12.0 K 2024-11-14T00:28:21,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/2c44e59652404d32aa76f3da28de9330 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/2c44e59652404d32aa76f3da28de9330 2024-11-14T00:28:21,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/2c44e59652404d32aa76f3da28de9330, entries=150, sequenceid=334, filesize=12.0 K 2024-11-14T00:28:21,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=20.13 KB/20610 for a2caf5d1c25d111104a7450b3aaca862 in 950ms, sequenceid=334, compaction requested=true 2024-11-14T00:28:21,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:21,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:21,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:21,516 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:21,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:21,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:21,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2caf5d1c25d111104a7450b3aaca862:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:21,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:21,516 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:21,517 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:21,518 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/B is initiating minor compaction (all files) 2024-11-14T00:28:21,518 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/B in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:21,518 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/f630eef01d294304bf9922feb7698807, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/7929338c69c74a1ebb0abb14632ca23b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/9dc25cc6f2ef424cb6453dfdaf5c989e] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=36.7 K 2024-11-14T00:28:21,518 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:21,518 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/A is initiating minor compaction (all files) 2024-11-14T00:28:21,518 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/A in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:21,518 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/fc03c55bbd914ba1bdf6cc052d901b29, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/ec0966de5c8c4a64878d3b6a666dafc2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/5d22b70aaba041678543085e7aa19bd5] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=92.2 K 2024-11-14T00:28:21,518 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:21,519 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/fc03c55bbd914ba1bdf6cc052d901b29, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/ec0966de5c8c4a64878d3b6a666dafc2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/5d22b70aaba041678543085e7aa19bd5] 2024-11-14T00:28:21,519 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting f630eef01d294304bf9922feb7698807, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731544096002 2024-11-14T00:28:21,519 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 7929338c69c74a1ebb0abb14632ca23b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731544098245 2024-11-14T00:28:21,519 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc03c55bbd914ba1bdf6cc052d901b29, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731544096002 2024-11-14T00:28:21,520 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dc25cc6f2ef424cb6453dfdaf5c989e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731544099425 2024-11-14T00:28:21,520 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec0966de5c8c4a64878d3b6a666dafc2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731544098245 2024-11-14T00:28:21,520 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d22b70aaba041678543085e7aa19bd5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731544099425 2024-11-14T00:28:21,577 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:21,579 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111457e0a93e192647e885bb31c61255945c_a2caf5d1c25d111104a7450b3aaca862 store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:21,581 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#B#compaction#156 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:21,581 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/efe2a3f00a6b4bcf80662918cfef2bae is 50, key is test_row_0/B:col10/1731544100565/Put/seqid=0 2024-11-14T00:28:21,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742019_1195 (size=13085) 2024-11-14T00:28:21,633 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111457e0a93e192647e885bb31c61255945c_a2caf5d1c25d111104a7450b3aaca862, store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:21,633 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111457e0a93e192647e885bb31c61255945c_a2caf5d1c25d111104a7450b3aaca862 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:21,642 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/efe2a3f00a6b4bcf80662918cfef2bae as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/efe2a3f00a6b4bcf80662918cfef2bae 2024-11-14T00:28:21,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742020_1196 (size=4469) 2024-11-14T00:28:21,658 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/B of a2caf5d1c25d111104a7450b3aaca862 into efe2a3f00a6b4bcf80662918cfef2bae(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:21,658 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:21,658 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/B, priority=13, startTime=1731544101516; duration=0sec 2024-11-14T00:28:21,658 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:21,658 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:B 2024-11-14T00:28:21,658 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:21,660 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:21,660 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a2caf5d1c25d111104a7450b3aaca862/C is initiating minor compaction (all files) 2024-11-14T00:28:21,660 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a2caf5d1c25d111104a7450b3aaca862/C in TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:21,660 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/822b75dbd53849d38a868e530e73cc3b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/ce040e9f82c74b83824f99a12207cb50, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/2c44e59652404d32aa76f3da28de9330] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp, totalSize=36.7 K 2024-11-14T00:28:21,661 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 822b75dbd53849d38a868e530e73cc3b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731544096002 2024-11-14T00:28:21,661 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting ce040e9f82c74b83824f99a12207cb50, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731544098245 2024-11-14T00:28:21,661 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c44e59652404d32aa76f3da28de9330, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731544099425 2024-11-14T00:28:21,678 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#C#compaction#158 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:21,678 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/1b1c68a49c084f2b9c7501cbad234e67 is 50, key is test_row_0/C:col10/1731544100565/Put/seqid=0 2024-11-14T00:28:21,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742021_1197 (size=13085) 2024-11-14T00:28:21,697 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/1b1c68a49c084f2b9c7501cbad234e67 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1b1c68a49c084f2b9c7501cbad234e67 2024-11-14T00:28:21,704 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/C of a2caf5d1c25d111104a7450b3aaca862 into 1b1c68a49c084f2b9c7501cbad234e67(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:21,705 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:21,705 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/C, priority=13, startTime=1731544101516; duration=0sec 2024-11-14T00:28:21,705 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:21,705 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:C 2024-11-14T00:28:22,058 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2caf5d1c25d111104a7450b3aaca862#A#compaction#157 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:22,059 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/cbb3c5f94fc0440f81711cc20997f48e is 175, key is test_row_0/A:col10/1731544100565/Put/seqid=0 2024-11-14T00:28:22,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742022_1198 (size=32039) 2024-11-14T00:28:22,471 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/cbb3c5f94fc0440f81711cc20997f48e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/cbb3c5f94fc0440f81711cc20997f48e 2024-11-14T00:28:22,483 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a2caf5d1c25d111104a7450b3aaca862/A of a2caf5d1c25d111104a7450b3aaca862 into cbb3c5f94fc0440f81711cc20997f48e(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:22,483 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a2caf5d1c25d111104a7450b3aaca862: 2024-11-14T00:28:22,483 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., storeName=a2caf5d1c25d111104a7450b3aaca862/A, priority=13, startTime=1731544101516; duration=0sec 2024-11-14T00:28:22,483 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:22,483 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2caf5d1c25d111104a7450b3aaca862:A 2024-11-14T00:28:23,206 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T00:28:26,456 INFO [Thread-541 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-541. 2024-11-14T00:28:26,457 DEBUG [Thread-541 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:26,457 DEBUG [Thread-541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:26,457 DEBUG [Thread-541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:26,457 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:26,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T00:28:26,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-14T00:28:26,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-14T00:28:26,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-14T00:28:26,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-14T00:28:26,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-14T00:28:26,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T00:28:26,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2228 2024-11-14T00:28:26,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2211 2024-11-14T00:28:26,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T00:28:26,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1026 2024-11-14T00:28:26,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3077 rows 2024-11-14T00:28:26,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1029 2024-11-14T00:28:26,457 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3086 rows 2024-11-14T00:28:26,457 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T00:28:26,457 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T00:28:26,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:26,458 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:26,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:26,458 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-14T00:28:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T00:28:26,462 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544106462"}]},"ts":"1731544106462"} 2024-11-14T00:28:26,465 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T00:28:26,465 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T00:28:26,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T00:28:26,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2caf5d1c25d111104a7450b3aaca862, UNASSIGN}] 2024-11-14T00:28:26,469 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2caf5d1c25d111104a7450b3aaca862, UNASSIGN 2024-11-14T00:28:26,470 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=a2caf5d1c25d111104a7450b3aaca862, regionState=CLOSING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:26,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2caf5d1c25d111104a7450b3aaca862, UNASSIGN because future has completed 2024-11-14T00:28:26,472 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T00:28:26,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:28:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T00:28:26,625 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(122): Close a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:26,626 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T00:28:26,626 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1722): Closing a2caf5d1c25d111104a7450b3aaca862, disabling compactions & flushes 2024-11-14T00:28:26,626 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:26,626 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:26,626 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. after waiting 0 ms 2024-11-14T00:28:26,626 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:26,626 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(2902): Flushing a2caf5d1c25d111104a7450b3aaca862 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-14T00:28:26,637 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145bca69209f284cce8b129817c79802a7_a2caf5d1c25d111104a7450b3aaca862 is 50, key is test_row_1/A:col10/1731544100576/Put/seqid=0 2024-11-14T00:28:26,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742023_1199 (size=9914) 2024-11-14T00:28:26,644 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:26,649 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145bca69209f284cce8b129817c79802a7_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145bca69209f284cce8b129817c79802a7_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:26,652 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/677becc2f24c497e81d681d83dbd492c, store: [table=TestAcidGuarantees family=A region=a2caf5d1c25d111104a7450b3aaca862] 2024-11-14T00:28:26,653 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/677becc2f24c497e81d681d83dbd492c is 175, key is test_row_1/A:col10/1731544100576/Put/seqid=0 2024-11-14T00:28:26,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742024_1200 (size=22561) 2024-11-14T00:28:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T00:28:27,057 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/677becc2f24c497e81d681d83dbd492c 2024-11-14T00:28:27,069 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/b48c65d4a43942af86e1424d77953dda is 50, key is test_row_1/B:col10/1731544100576/Put/seqid=0 2024-11-14T00:28:27,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742025_1201 (size=9857) 2024-11-14T00:28:27,074 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/b48c65d4a43942af86e1424d77953dda 2024-11-14T00:28:27,082 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/dc053354bf734eb4912ee9424f7ae4a4 is 50, key is test_row_1/C:col10/1731544100576/Put/seqid=0 2024-11-14T00:28:27,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T00:28:27,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742026_1202 (size=9857) 2024-11-14T00:28:27,093 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/dc053354bf734eb4912ee9424f7ae4a4 2024-11-14T00:28:27,101 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/A/677becc2f24c497e81d681d83dbd492c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/677becc2f24c497e81d681d83dbd492c 2024-11-14T00:28:27,107 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/677becc2f24c497e81d681d83dbd492c, entries=100, sequenceid=344, filesize=22.0 K 2024-11-14T00:28:27,109 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/B/b48c65d4a43942af86e1424d77953dda as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/b48c65d4a43942af86e1424d77953dda 2024-11-14T00:28:27,114 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/b48c65d4a43942af86e1424d77953dda, entries=100, sequenceid=344, filesize=9.6 K 2024-11-14T00:28:27,115 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/.tmp/C/dc053354bf734eb4912ee9424f7ae4a4 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/dc053354bf734eb4912ee9424f7ae4a4 2024-11-14T00:28:27,120 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/dc053354bf734eb4912ee9424f7ae4a4, entries=100, sequenceid=344, filesize=9.6 K 2024-11-14T00:28:27,121 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for a2caf5d1c25d111104a7450b3aaca862 in 495ms, sequenceid=344, compaction requested=false 2024-11-14T00:28:27,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/13afcc7f0d364fd5acf02a68c2e7a634, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/2013b42656da4f5997614b8fd081a0af, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/2aa166c0c5354ec6a747d5ee53bc8e8c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/ea59b97f2d7d412983079a617f623629, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/24d02dd4913f4556b9e71dc8f72b514f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/00461c02e34748e58ff997c9c14664d1, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/8eb547e24e8b4e46bc91139f917b5763, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/04982b8567c842679a294094f9913038, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/244312e34c6045a3a144cf1d04f66e56, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/fe4ad4171d2849b08a4d862c6766661a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/b345316a8c68446a8baee5b021341efb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/e6547521538046c5a70b991b742ec886, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/744d26f646a6487e84cfbf68a5f2d5de, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/8935f4cdcdfa48088203019098f6fa2e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/629d89b91619420ab1f8fb6b448f02f6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/dc357625bc3d493fa09317b00d46fda4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/478a0fb5ad734ef8b17868c16bc66295, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/aff8b62f23d140749e267a9d39cbe183, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/0f813ff3e08847fd9a2cd34bab60bec7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/a62efab2860d4dcca7c187c2114e86ba, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/fc03c55bbd914ba1bdf6cc052d901b29, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/ec0966de5c8c4a64878d3b6a666dafc2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/5d22b70aaba041678543085e7aa19bd5] to archive 2024-11-14T00:28:27,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:28:27,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/13afcc7f0d364fd5acf02a68c2e7a634 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/13afcc7f0d364fd5acf02a68c2e7a634 2024-11-14T00:28:27,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/2013b42656da4f5997614b8fd081a0af to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/2013b42656da4f5997614b8fd081a0af 2024-11-14T00:28:27,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/2aa166c0c5354ec6a747d5ee53bc8e8c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/2aa166c0c5354ec6a747d5ee53bc8e8c 2024-11-14T00:28:27,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/ea59b97f2d7d412983079a617f623629 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/ea59b97f2d7d412983079a617f623629 2024-11-14T00:28:27,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/24d02dd4913f4556b9e71dc8f72b514f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/24d02dd4913f4556b9e71dc8f72b514f 2024-11-14T00:28:27,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/00461c02e34748e58ff997c9c14664d1 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/00461c02e34748e58ff997c9c14664d1 2024-11-14T00:28:27,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/8eb547e24e8b4e46bc91139f917b5763 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/8eb547e24e8b4e46bc91139f917b5763 2024-11-14T00:28:27,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/04982b8567c842679a294094f9913038 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/04982b8567c842679a294094f9913038 2024-11-14T00:28:27,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/244312e34c6045a3a144cf1d04f66e56 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/244312e34c6045a3a144cf1d04f66e56 2024-11-14T00:28:27,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/fe4ad4171d2849b08a4d862c6766661a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/fe4ad4171d2849b08a4d862c6766661a 2024-11-14T00:28:27,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/b345316a8c68446a8baee5b021341efb to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/b345316a8c68446a8baee5b021341efb 2024-11-14T00:28:27,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/e6547521538046c5a70b991b742ec886 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/e6547521538046c5a70b991b742ec886 2024-11-14T00:28:27,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/744d26f646a6487e84cfbf68a5f2d5de to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/744d26f646a6487e84cfbf68a5f2d5de 2024-11-14T00:28:27,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/8935f4cdcdfa48088203019098f6fa2e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/8935f4cdcdfa48088203019098f6fa2e 2024-11-14T00:28:27,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/629d89b91619420ab1f8fb6b448f02f6 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/629d89b91619420ab1f8fb6b448f02f6 2024-11-14T00:28:27,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/dc357625bc3d493fa09317b00d46fda4 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/dc357625bc3d493fa09317b00d46fda4 2024-11-14T00:28:27,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/478a0fb5ad734ef8b17868c16bc66295 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/478a0fb5ad734ef8b17868c16bc66295 2024-11-14T00:28:27,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/aff8b62f23d140749e267a9d39cbe183 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/aff8b62f23d140749e267a9d39cbe183 2024-11-14T00:28:27,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/0f813ff3e08847fd9a2cd34bab60bec7 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/0f813ff3e08847fd9a2cd34bab60bec7 2024-11-14T00:28:27,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/a62efab2860d4dcca7c187c2114e86ba to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/a62efab2860d4dcca7c187c2114e86ba 2024-11-14T00:28:27,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/fc03c55bbd914ba1bdf6cc052d901b29 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/fc03c55bbd914ba1bdf6cc052d901b29 2024-11-14T00:28:27,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/ec0966de5c8c4a64878d3b6a666dafc2 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/ec0966de5c8c4a64878d3b6a666dafc2 2024-11-14T00:28:27,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/5d22b70aaba041678543085e7aa19bd5 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/5d22b70aaba041678543085e7aa19bd5 2024-11-14T00:28:27,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/540d2c66e0024fbfae450647e5deb7c5, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/6a287f1466f04b12bdc14b24d1086ee4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/41c157b8a7574bd49a5996b806019a6c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/4548daece36f4185b46b882ad74a44d3, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/b6697dd5f67e473f9857d299f8874a71, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/b28de671374443acb15e8ab0174e46c1, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/655233bce130438da2617761bdbe3d2c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/d1c02a080fc24518b7f26e37922a7bfe, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/f7e47a0721f6452b863caf20e3c8d8b0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/e830214dccd147738e52180ac6d2b5c9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/4fcd0c8bbccf4707af545ee965a58ae8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/2e83ad7b7e23496eaf07dc17132fdcaf, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/c0092b083a264801ad7c17dfd2441027, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/d92a3193229f4d8981b66a6b47ce13fb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/7f6041dc63904605b478476f7540cd22, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/0a593ac3387f4ef48a2d9b49998f9dc0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/8f1b7651bc4a4740a2044306fb675017, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/66188815ebc54eee84dca59e7aa31a3f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/bf2d95c322ef463c812e92672bcb1322, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/f630eef01d294304bf9922feb7698807, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/9768bf75838940a0b90686f4e4014402, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/7929338c69c74a1ebb0abb14632ca23b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/9dc25cc6f2ef424cb6453dfdaf5c989e] to archive 2024-11-14T00:28:27,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:28:27,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/540d2c66e0024fbfae450647e5deb7c5 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/540d2c66e0024fbfae450647e5deb7c5 2024-11-14T00:28:27,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/6a287f1466f04b12bdc14b24d1086ee4 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/6a287f1466f04b12bdc14b24d1086ee4 2024-11-14T00:28:27,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/41c157b8a7574bd49a5996b806019a6c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/41c157b8a7574bd49a5996b806019a6c 2024-11-14T00:28:27,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/4548daece36f4185b46b882ad74a44d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/4548daece36f4185b46b882ad74a44d3 2024-11-14T00:28:27,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/b6697dd5f67e473f9857d299f8874a71 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/b6697dd5f67e473f9857d299f8874a71 2024-11-14T00:28:27,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/b28de671374443acb15e8ab0174e46c1 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/b28de671374443acb15e8ab0174e46c1 2024-11-14T00:28:27,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/655233bce130438da2617761bdbe3d2c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/655233bce130438da2617761bdbe3d2c 2024-11-14T00:28:27,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/d1c02a080fc24518b7f26e37922a7bfe to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/d1c02a080fc24518b7f26e37922a7bfe 2024-11-14T00:28:27,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/f7e47a0721f6452b863caf20e3c8d8b0 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/f7e47a0721f6452b863caf20e3c8d8b0 2024-11-14T00:28:27,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/e830214dccd147738e52180ac6d2b5c9 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/e830214dccd147738e52180ac6d2b5c9 2024-11-14T00:28:27,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/4fcd0c8bbccf4707af545ee965a58ae8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/4fcd0c8bbccf4707af545ee965a58ae8 2024-11-14T00:28:27,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/2e83ad7b7e23496eaf07dc17132fdcaf to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/2e83ad7b7e23496eaf07dc17132fdcaf 2024-11-14T00:28:27,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/c0092b083a264801ad7c17dfd2441027 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/c0092b083a264801ad7c17dfd2441027 2024-11-14T00:28:27,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/d92a3193229f4d8981b66a6b47ce13fb to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/d92a3193229f4d8981b66a6b47ce13fb 2024-11-14T00:28:27,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/7f6041dc63904605b478476f7540cd22 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/7f6041dc63904605b478476f7540cd22 2024-11-14T00:28:27,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/0a593ac3387f4ef48a2d9b49998f9dc0 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/0a593ac3387f4ef48a2d9b49998f9dc0 2024-11-14T00:28:27,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/8f1b7651bc4a4740a2044306fb675017 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/8f1b7651bc4a4740a2044306fb675017 2024-11-14T00:28:27,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/66188815ebc54eee84dca59e7aa31a3f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/66188815ebc54eee84dca59e7aa31a3f 2024-11-14T00:28:27,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/bf2d95c322ef463c812e92672bcb1322 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/bf2d95c322ef463c812e92672bcb1322 2024-11-14T00:28:27,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/f630eef01d294304bf9922feb7698807 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/f630eef01d294304bf9922feb7698807 2024-11-14T00:28:27,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/9768bf75838940a0b90686f4e4014402 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/9768bf75838940a0b90686f4e4014402 2024-11-14T00:28:27,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/7929338c69c74a1ebb0abb14632ca23b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/7929338c69c74a1ebb0abb14632ca23b 2024-11-14T00:28:27,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/9dc25cc6f2ef424cb6453dfdaf5c989e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/9dc25cc6f2ef424cb6453dfdaf5c989e 2024-11-14T00:28:27,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/62c8077ff9e14fa387ae99596c19acd4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/f20a76de5d7a4dc4b528b092efa803ef, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/9e8ca91cabed42d4a7117aac5efca04f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/20b5650261414f62a154d4656aa107d7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/c677356e016c44c08692215b3cc08b0b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/b6272871c2b54c6b8cd78203204905c5, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/f52791da0f054bed9c03d789ebac8ffe, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/0f3a39220fe340dfac55a9c9109bf621, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/e6c59268de854064bdf359fdac5e2493, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1350ea79bf7246d28369b579dc1fc01a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/64092ab7132d421ab422eef2f9f5d546, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1a5c2c36e88a4ad588d2030c5dd77e5d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/6fc660f6dd1646edb7040b0df230c5d1, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/3824bc883edf470c973c71d7aaf94b51, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/bddf8d214eb140b29cb824de857b397e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/ac9120b2bd3d449bb7074fb693e73c63, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/a2eb0381d5324376be7659beb04c25d5, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1c76fc77f09743749f680af57d4aa088, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/c89c3e4c181d4a6d847d43ae93b14c0a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/822b75dbd53849d38a868e530e73cc3b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/6bdada454cd043d1809fe6ae5680447b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/ce040e9f82c74b83824f99a12207cb50, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/2c44e59652404d32aa76f3da28de9330] to archive 2024-11-14T00:28:27,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:28:27,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/62c8077ff9e14fa387ae99596c19acd4 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/62c8077ff9e14fa387ae99596c19acd4 2024-11-14T00:28:27,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/f20a76de5d7a4dc4b528b092efa803ef to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/f20a76de5d7a4dc4b528b092efa803ef 2024-11-14T00:28:27,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/9e8ca91cabed42d4a7117aac5efca04f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/9e8ca91cabed42d4a7117aac5efca04f 2024-11-14T00:28:27,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/20b5650261414f62a154d4656aa107d7 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/20b5650261414f62a154d4656aa107d7 2024-11-14T00:28:27,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/c677356e016c44c08692215b3cc08b0b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/c677356e016c44c08692215b3cc08b0b 2024-11-14T00:28:27,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/b6272871c2b54c6b8cd78203204905c5 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/b6272871c2b54c6b8cd78203204905c5 2024-11-14T00:28:27,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/f52791da0f054bed9c03d789ebac8ffe to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/f52791da0f054bed9c03d789ebac8ffe 2024-11-14T00:28:27,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/0f3a39220fe340dfac55a9c9109bf621 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/0f3a39220fe340dfac55a9c9109bf621 2024-11-14T00:28:27,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/e6c59268de854064bdf359fdac5e2493 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/e6c59268de854064bdf359fdac5e2493 2024-11-14T00:28:27,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1350ea79bf7246d28369b579dc1fc01a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1350ea79bf7246d28369b579dc1fc01a 2024-11-14T00:28:27,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/64092ab7132d421ab422eef2f9f5d546 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/64092ab7132d421ab422eef2f9f5d546 2024-11-14T00:28:27,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1a5c2c36e88a4ad588d2030c5dd77e5d to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1a5c2c36e88a4ad588d2030c5dd77e5d 2024-11-14T00:28:27,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/6fc660f6dd1646edb7040b0df230c5d1 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/6fc660f6dd1646edb7040b0df230c5d1 2024-11-14T00:28:27,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/3824bc883edf470c973c71d7aaf94b51 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/3824bc883edf470c973c71d7aaf94b51 2024-11-14T00:28:27,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/bddf8d214eb140b29cb824de857b397e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/bddf8d214eb140b29cb824de857b397e 2024-11-14T00:28:27,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/ac9120b2bd3d449bb7074fb693e73c63 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/ac9120b2bd3d449bb7074fb693e73c63 2024-11-14T00:28:27,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/a2eb0381d5324376be7659beb04c25d5 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/a2eb0381d5324376be7659beb04c25d5 2024-11-14T00:28:27,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1c76fc77f09743749f680af57d4aa088 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1c76fc77f09743749f680af57d4aa088 2024-11-14T00:28:27,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/c89c3e4c181d4a6d847d43ae93b14c0a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/c89c3e4c181d4a6d847d43ae93b14c0a 2024-11-14T00:28:27,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/822b75dbd53849d38a868e530e73cc3b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/822b75dbd53849d38a868e530e73cc3b 2024-11-14T00:28:27,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/6bdada454cd043d1809fe6ae5680447b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/6bdada454cd043d1809fe6ae5680447b 2024-11-14T00:28:27,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/ce040e9f82c74b83824f99a12207cb50 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/ce040e9f82c74b83824f99a12207cb50 2024-11-14T00:28:27,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/2c44e59652404d32aa76f3da28de9330 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/2c44e59652404d32aa76f3da28de9330 2024-11-14T00:28:27,284 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/recovered.edits/347.seqid, newMaxSeqId=347, maxSeqId=4 2024-11-14T00:28:27,285 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. 2024-11-14T00:28:27,285 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] regionserver.HRegion(1676): Region close journal for a2caf5d1c25d111104a7450b3aaca862: Waiting for close lock at 1731544106626Running coprocessor pre-close hooks at 1731544106626Disabling compacts and flushes for region at 1731544106626Disabling writes for close at 1731544106626Obtaining lock to block concurrent updates at 1731544106626Preparing flush snapshotting stores in a2caf5d1c25d111104a7450b3aaca862 at 1731544106626Finished memstore snapshotting TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731544106626Flushing stores of TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862. at 1731544106627 (+1 ms)Flushing a2caf5d1c25d111104a7450b3aaca862/A: creating writer at 1731544106627Flushing mob file a2caf5d1c25d111104a7450b3aaca862/A: appending metadata at 1731544106637 (+10 ms)Flushing mob file a2caf5d1c25d111104a7450b3aaca862/A: closing flushed file at 1731544106637Flushing a2caf5d1c25d111104a7450b3aaca862/A: appending metadata at 1731544106652 (+15 ms)Flushing a2caf5d1c25d111104a7450b3aaca862/A: closing flushed file at 1731544106652Flushing a2caf5d1c25d111104a7450b3aaca862/B: creating writer at 1731544107062 (+410 ms)Flushing a2caf5d1c25d111104a7450b3aaca862/B: appending metadata at 1731544107069 (+7 ms)Flushing a2caf5d1c25d111104a7450b3aaca862/B: closing flushed file at 1731544107069Flushing a2caf5d1c25d111104a7450b3aaca862/C: creating writer at 1731544107078 (+9 ms)Flushing a2caf5d1c25d111104a7450b3aaca862/C: appending metadata at 1731544107081 (+3 ms)Flushing a2caf5d1c25d111104a7450b3aaca862/C: closing flushed file at 1731544107082 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@724d9920: reopening flushed file at 1731544107101 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d39d4d9: reopening flushed file at 1731544107107 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bd52b29: reopening flushed file at 1731544107115 (+8 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for a2caf5d1c25d111104a7450b3aaca862 in 495ms, sequenceid=344, compaction requested=false at 1731544107121 (+6 ms)Writing region close event to WAL at 1731544107280 (+159 ms)Running coprocessor post-close hooks at 1731544107285 (+5 ms)Closed at 1731544107285 2024-11-14T00:28:27,289 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=51}] handler.UnassignRegionHandler(157): Closed a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,290 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=a2caf5d1c25d111104a7450b3aaca862, regionState=CLOSED 2024-11-14T00:28:27,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=50, state=RUNNABLE, hasLock=false; CloseRegionProcedure a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:28:27,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-14T00:28:27,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseRegionProcedure a2caf5d1c25d111104a7450b3aaca862, server=63baa11b75f3,33199,1731544045387 in 822 msec 2024-11-14T00:28:27,298 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-11-14T00:28:27,298 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2caf5d1c25d111104a7450b3aaca862, UNASSIGN in 828 msec 2024-11-14T00:28:27,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-14T00:28:27,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 833 msec 2024-11-14T00:28:27,303 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544107303"}]},"ts":"1731544107303"} 2024-11-14T00:28:27,305 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T00:28:27,305 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T00:28:27,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 847 msec 2024-11-14T00:28:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T00:28:27,596 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:27,596 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-14T00:28:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:27,598 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=52, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T00:28:27,599 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=52, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:27,601 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,604 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/recovered.edits] 2024-11-14T00:28:27,606 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/677becc2f24c497e81d681d83dbd492c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/677becc2f24c497e81d681d83dbd492c 2024-11-14T00:28:27,607 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/cbb3c5f94fc0440f81711cc20997f48e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/A/cbb3c5f94fc0440f81711cc20997f48e 2024-11-14T00:28:27,610 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/b48c65d4a43942af86e1424d77953dda to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/b48c65d4a43942af86e1424d77953dda 2024-11-14T00:28:27,611 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/efe2a3f00a6b4bcf80662918cfef2bae to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/B/efe2a3f00a6b4bcf80662918cfef2bae 2024-11-14T00:28:27,614 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1b1c68a49c084f2b9c7501cbad234e67 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/1b1c68a49c084f2b9c7501cbad234e67 2024-11-14T00:28:27,615 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/dc053354bf734eb4912ee9424f7ae4a4 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/C/dc053354bf734eb4912ee9424f7ae4a4 2024-11-14T00:28:27,618 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/recovered.edits/347.seqid to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862/recovered.edits/347.seqid 2024-11-14T00:28:27,619 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,619 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T00:28:27,619 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T00:28:27,620 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T00:28:27,625 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114036264b7f72b46ef891305bb5039a4b2_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114036264b7f72b46ef891305bb5039a4b2_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,626 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111403ed005ec14d42eb97d04ba856406694_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111403ed005ec14d42eb97d04ba856406694_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,627 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142ec3de2d84584712acabbab0259df0cc_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142ec3de2d84584712acabbab0259df0cc_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,629 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143249bd84a1b34e4a821395e28c68d2fc_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143249bd84a1b34e4a821395e28c68d2fc_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,630 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144e20784905d74ea9b8ad614c856ed8ce_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144e20784905d74ea9b8ad614c856ed8ce_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,631 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111452102f0d66b64c1eb039335ad6f5901c_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111452102f0d66b64c1eb039335ad6f5901c_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,632 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145725aabe6418405799851fe383e19ec1_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145725aabe6418405799851fe383e19ec1_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,634 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145bca69209f284cce8b129817c79802a7_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145bca69209f284cce8b129817c79802a7_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,635 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111465f4572b9f3247f486422735afdbc1fa_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111465f4572b9f3247f486422735afdbc1fa_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,636 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111493660e8b3ff64e7584f62401e40f496a_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111493660e8b3ff64e7584f62401e40f496a_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,638 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111494d0ada3fce048888c56256a8ba1baa9_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111494d0ada3fce048888c56256a8ba1baa9_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,639 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a534db2733cd4406acd3d8b4843e4d3a_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a534db2733cd4406acd3d8b4843e4d3a_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,640 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c1b0c9aa69494593bfff8b16f78c510e_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c1b0c9aa69494593bfff8b16f78c510e_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,641 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c415cd38326b4d25a46ad622359c6fad_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c415cd38326b4d25a46ad622359c6fad_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,643 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c78bb357019e4c7c93b68426d758df89_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c78bb357019e4c7c93b68426d758df89_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,644 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cf8e634961924299a081187a52d760bf_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cf8e634961924299a081187a52d760bf_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,646 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d272dcb7330c4f4b962976ceee92d03c_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d272dcb7330c4f4b962976ceee92d03c_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,648 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114daacf1709d4e441c81a4dc65b706c622_a2caf5d1c25d111104a7450b3aaca862 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114daacf1709d4e441c81a4dc65b706c622_a2caf5d1c25d111104a7450b3aaca862 2024-11-14T00:28:27,649 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T00:28:27,652 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=52, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:27,654 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T00:28:27,656 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T00:28:27,660 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=52, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:27,661 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T00:28:27,661 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731544107661"}]},"ts":"9223372036854775807"} 2024-11-14T00:28:27,664 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T00:28:27,664 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a2caf5d1c25d111104a7450b3aaca862, NAME => 'TestAcidGuarantees,,1731544077575.a2caf5d1c25d111104a7450b3aaca862.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T00:28:27,664 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T00:28:27,664 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731544107664"}]},"ts":"9223372036854775807"} 2024-11-14T00:28:27,666 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T00:28:27,666 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=52, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:27,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-11-14T00:28:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T00:28:27,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T00:28:27,706 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:27,714 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237 (was 238), OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1210 (was 1301), ProcessCount=11 (was 11), AvailableMemoryMB=1669 (was 1902) 2024-11-14T00:28:27,724 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=237, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=1210, ProcessCount=11, AvailableMemoryMB=1669 2024-11-14T00:28:27,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T00:28:27,726 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T00:28:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:27,728 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T00:28:27,728 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:27,728 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 53 2024-11-14T00:28:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-14T00:28:27,729 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T00:28:27,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742027_1203 (size=959) 2024-11-14T00:28:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-14T00:28:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-14T00:28:28,143 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c 2024-11-14T00:28:28,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742028_1204 (size=53) 2024-11-14T00:28:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-14T00:28:28,559 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:28:28,559 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 328b146c51a69a897800000a78ffe3a7, disabling compactions & flushes 2024-11-14T00:28:28,559 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:28,559 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:28,559 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. after waiting 0 ms 2024-11-14T00:28:28,559 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:28,559 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:28,559 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 328b146c51a69a897800000a78ffe3a7: Waiting for close lock at 1731544108559Disabling compacts and flushes for region at 1731544108559Disabling writes for close at 1731544108559Writing region close event to WAL at 1731544108559Closed at 1731544108559 2024-11-14T00:28:28,560 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T00:28:28,561 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731544108560"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731544108560"}]},"ts":"1731544108560"} 2024-11-14T00:28:28,562 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T00:28:28,563 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T00:28:28,563 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544108563"}]},"ts":"1731544108563"} 2024-11-14T00:28:28,565 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T00:28:28,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=328b146c51a69a897800000a78ffe3a7, ASSIGN}] 2024-11-14T00:28:28,566 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=328b146c51a69a897800000a78ffe3a7, ASSIGN 2024-11-14T00:28:28,567 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=328b146c51a69a897800000a78ffe3a7, ASSIGN; state=OFFLINE, location=63baa11b75f3,33199,1731544045387; forceNewPlan=false, retain=false 2024-11-14T00:28:28,718 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=328b146c51a69a897800000a78ffe3a7, regionState=OPENING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:28,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=328b146c51a69a897800000a78ffe3a7, ASSIGN because future has completed 2024-11-14T00:28:28,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure 328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:28:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-14T00:28:28,876 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:28,876 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} 2024-11-14T00:28:28,876 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:28,876 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:28:28,876 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:28,876 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:28,878 INFO [StoreOpener-328b146c51a69a897800000a78ffe3a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:28,880 INFO [StoreOpener-328b146c51a69a897800000a78ffe3a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 328b146c51a69a897800000a78ffe3a7 columnFamilyName A 2024-11-14T00:28:28,880 DEBUG [StoreOpener-328b146c51a69a897800000a78ffe3a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:28,881 INFO [StoreOpener-328b146c51a69a897800000a78ffe3a7-1 {}] regionserver.HStore(327): Store=328b146c51a69a897800000a78ffe3a7/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:28:28,881 INFO [StoreOpener-328b146c51a69a897800000a78ffe3a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:28,882 INFO [StoreOpener-328b146c51a69a897800000a78ffe3a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 328b146c51a69a897800000a78ffe3a7 columnFamilyName B 2024-11-14T00:28:28,882 DEBUG [StoreOpener-328b146c51a69a897800000a78ffe3a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:28,883 INFO [StoreOpener-328b146c51a69a897800000a78ffe3a7-1 {}] regionserver.HStore(327): Store=328b146c51a69a897800000a78ffe3a7/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:28:28,883 INFO [StoreOpener-328b146c51a69a897800000a78ffe3a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:28,886 INFO [StoreOpener-328b146c51a69a897800000a78ffe3a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 328b146c51a69a897800000a78ffe3a7 columnFamilyName C 2024-11-14T00:28:28,886 DEBUG [StoreOpener-328b146c51a69a897800000a78ffe3a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:28,887 INFO [StoreOpener-328b146c51a69a897800000a78ffe3a7-1 {}] regionserver.HStore(327): Store=328b146c51a69a897800000a78ffe3a7/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:28:28,887 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:28,888 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:28,888 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:28,889 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:28,889 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:28,890 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T00:28:28,891 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:28,893 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T00:28:28,894 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 328b146c51a69a897800000a78ffe3a7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74606361, jitterRate=0.11172141134738922}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T00:28:28,894 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:28,895 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 328b146c51a69a897800000a78ffe3a7: Running coprocessor pre-open hook at 1731544108877Writing region info on filesystem at 1731544108877Initializing all the Stores at 1731544108877Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544108878 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544108878Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544108878Cleaning up temporary data from old regions at 1731544108889 (+11 ms)Running coprocessor post-open hooks at 1731544108894 (+5 ms)Region opened successfully at 1731544108895 (+1 ms) 2024-11-14T00:28:28,896 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., pid=55, masterSystemTime=1731544108872 2024-11-14T00:28:28,898 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:28,898 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:28,899 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=328b146c51a69a897800000a78ffe3a7, regionState=OPEN, openSeqNum=2, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:28,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; OpenRegionProcedure 328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:28:28,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-14T00:28:28,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; OpenRegionProcedure 328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 in 181 msec 2024-11-14T00:28:28,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-14T00:28:28,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=328b146c51a69a897800000a78ffe3a7, ASSIGN in 338 msec 2024-11-14T00:28:28,906 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T00:28:28,907 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544108907"}]},"ts":"1731544108907"} 2024-11-14T00:28:28,909 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T00:28:28,910 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=53, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T00:28:28,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-11-14T00:28:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-11-14T00:28:29,875 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:29,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dccd3b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:29,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:29,877 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58048424, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:29,879 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:29,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fbd7641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,880 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:29,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:29,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:29,882 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:29,882 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T00:28:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 63baa11b75f3,36271,1731544044347 2024-11-14T00:28:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1db4879f 2024-11-14T00:28:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T00:28:29,884 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T00:28:29,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fff55e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,885 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:29,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:29,886 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:29,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:29,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:29,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65622206, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:29,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:29,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:29,887 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:29,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ba51409, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8800d1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,888 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:29,888 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:29,889 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:29,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:29,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:29,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d6f1727, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,889 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:29,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:29,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:29,890 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:29,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cb27b07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14273c65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,891 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:29,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:29,892 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7deb7c25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:29,893 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:29,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7753605d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26d1d48c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:29,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:29,895 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:29,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:29,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:29,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e1b645b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:29,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:29,897 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ae9541, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2390fab3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,898 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:29,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:29,899 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eb400f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:29,905 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:29,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62d0624f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc79077, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:29,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:29,910 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:29,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6220191b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:29,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:29,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:29,912 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:29,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef1e2fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@397178fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:29,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:29,914 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:29,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:29,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:29,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73ecc23b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:29,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:29,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:29,915 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:29,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ff71ce2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c443c4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:29,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:29,917 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11dfeba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:29,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:29,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:29,918 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:29,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38dbad99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59065d79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,920 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:29,920 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:29,920 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10c33dd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:29,922 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:29,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c091c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@618e8185, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:29,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:29,924 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d56c51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:29,924 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:29,927 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bc91ca3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:29,928 DEBUG [Thread-978 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:29,928 DEBUG [Thread-987 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:29,929 DEBUG [Thread-993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:29,929 DEBUG [Thread-989 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:29,930 DEBUG [Thread-985 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:29,931 DEBUG [Thread-991 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:29,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:29,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:29,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:29,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51574, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:29,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:29,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:29,933 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:29,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:29,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:28:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:28:29,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:28:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:29,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:28:29,939 DEBUG [Thread-982 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:29,939 DEBUG [Thread-980 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:29,940 DEBUG [Thread-976 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:29,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:29,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:29,941 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:29,941 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:29,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:29,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-14T00:28:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T00:28:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:29,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:29,944 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:29,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:29,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:28:29,948 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:29,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:28:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:28:29,950 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:29,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:28:29,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:28:29,956 DEBUG [Thread-974 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:29,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:29,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:28:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:29,968 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:29,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T00:28:30,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/00e2c348637e4272856f71dc8bc3cf2d is 50, key is test_row_0/A:col10/1731544109965/Put/seqid=0 2024-11-14T00:28:30,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544170017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544170016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544170021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544170030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544170030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T00:28:30,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742029_1205 (size=12001) 2024-11-14T00:28:30,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T00:28:30,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:30,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:30,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:30,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:30,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:30,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544170136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544170136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544170136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544170145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544170160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T00:28:30,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T00:28:30,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:30,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:30,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:30,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:30,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:30,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544170346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544170346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544170346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544170360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544170377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T00:28:30,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:30,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:30,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:30,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:30,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:30,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/00e2c348637e4272856f71dc8bc3cf2d 2024-11-14T00:28:30,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/9a38c8b8998349f7b3f54fac21190878 is 50, key is test_row_0/B:col10/1731544109965/Put/seqid=0 2024-11-14T00:28:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T00:28:30,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T00:28:30,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:30,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:30,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:30,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:30,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:30,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742030_1206 (size=12001) 2024-11-14T00:28:30,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/9a38c8b8998349f7b3f54fac21190878 2024-11-14T00:28:30,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/e83950b6268b4baf8d22c74035583a72 is 50, key is test_row_0/C:col10/1731544109965/Put/seqid=0 2024-11-14T00:28:30,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544170655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544170656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544170656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742031_1207 (size=12001) 2024-11-14T00:28:30,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/e83950b6268b4baf8d22c74035583a72 2024-11-14T00:28:30,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544170675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544170689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:30,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/00e2c348637e4272856f71dc8bc3cf2d as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/00e2c348637e4272856f71dc8bc3cf2d 2024-11-14T00:28:30,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/00e2c348637e4272856f71dc8bc3cf2d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T00:28:30,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/9a38c8b8998349f7b3f54fac21190878 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/9a38c8b8998349f7b3f54fac21190878 2024-11-14T00:28:30,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/9a38c8b8998349f7b3f54fac21190878, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T00:28:30,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/e83950b6268b4baf8d22c74035583a72 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e83950b6268b4baf8d22c74035583a72 2024-11-14T00:28:30,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T00:28:30,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:30,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:30,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:30,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:30,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:30,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e83950b6268b4baf8d22c74035583a72, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T00:28:30,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 328b146c51a69a897800000a78ffe3a7 in 809ms, sequenceid=14, compaction requested=false 2024-11-14T00:28:30,778 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T00:28:30,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:30,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T00:28:30,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:30,889 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T00:28:30,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/7e9f9ab83d3c41aeb3320c9cd0ae9bb9 is 50, key is test_row_0/A:col10/1731544110028/Put/seqid=0 2024-11-14T00:28:30,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742032_1208 (size=12001) 2024-11-14T00:28:30,948 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/7e9f9ab83d3c41aeb3320c9cd0ae9bb9 2024-11-14T00:28:30,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/1606cd57cfc24183a3e37c302002947b is 50, key is test_row_0/B:col10/1731544110028/Put/seqid=0 2024-11-14T00:28:30,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742033_1209 (size=12001) 2024-11-14T00:28:30,993 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/1606cd57cfc24183a3e37c302002947b 2024-11-14T00:28:31,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/53c275da8dc749d8b97888754923af5b is 50, key is test_row_0/C:col10/1731544110028/Put/seqid=0 2024-11-14T00:28:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T00:28:31,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742034_1210 (size=12001) 2024-11-14T00:28:31,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:31,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544171220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544171223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544171223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544171224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544171224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544171326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544171335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544171336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544171336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544171336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,532 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/53c275da8dc749d8b97888754923af5b 2024-11-14T00:28:31,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544171535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544171545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544171545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/7e9f9ab83d3c41aeb3320c9cd0ae9bb9 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/7e9f9ab83d3c41aeb3320c9cd0ae9bb9 2024-11-14T00:28:31,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544171546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544171548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,560 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/7e9f9ab83d3c41aeb3320c9cd0ae9bb9, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T00:28:31,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/1606cd57cfc24183a3e37c302002947b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/1606cd57cfc24183a3e37c302002947b 2024-11-14T00:28:31,570 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/1606cd57cfc24183a3e37c302002947b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T00:28:31,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/53c275da8dc749d8b97888754923af5b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/53c275da8dc749d8b97888754923af5b 2024-11-14T00:28:31,578 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/53c275da8dc749d8b97888754923af5b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T00:28:31,580 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 328b146c51a69a897800000a78ffe3a7 in 692ms, sequenceid=37, compaction requested=false 2024-11-14T00:28:31,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:31,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:31,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-14T00:28:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-14T00:28:31,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-14T00:28:31,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6420 sec 2024-11-14T00:28:31,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.6560 sec 2024-11-14T00:28:31,850 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:31,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T00:28:31,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:31,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/111994165bf64b7b84bc98007f83ea41 is 50, key is test_row_0/A:col10/1731544111220/Put/seqid=0 2024-11-14T00:28:31,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742035_1211 (size=14341) 2024-11-14T00:28:31,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/111994165bf64b7b84bc98007f83ea41 2024-11-14T00:28:31,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/f7f7cdf3ba2c4f54a2ff0112b7031c41 is 50, key is test_row_0/B:col10/1731544111220/Put/seqid=0 2024-11-14T00:28:31,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742036_1212 (size=12001) 2024-11-14T00:28:31,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544171938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544171939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544171946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544171948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544171949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:31,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:31,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544172056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544172066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544172066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544172068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544172077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T00:28:32,089 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:32,090 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:32,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-14T00:28:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T00:28:32,100 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:32,102 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:32,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T00:28:32,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-14T00:28:32,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:32,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:32,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:32,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:32,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:32,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544172266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544172280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544172280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544172286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544172300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/f7f7cdf3ba2c4f54a2ff0112b7031c41 2024-11-14T00:28:32,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/21b9a8f68dbd42fe891bdd0dae2a3058 is 50, key is test_row_0/C:col10/1731544111220/Put/seqid=0 2024-11-14T00:28:32,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742037_1213 (size=12001) 2024-11-14T00:28:32,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/21b9a8f68dbd42fe891bdd0dae2a3058 2024-11-14T00:28:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T00:28:32,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-14T00:28:32,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:32,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:32,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:32,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:32,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:32,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/111994165bf64b7b84bc98007f83ea41 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/111994165bf64b7b84bc98007f83ea41 2024-11-14T00:28:32,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/111994165bf64b7b84bc98007f83ea41, entries=200, sequenceid=52, filesize=14.0 K 2024-11-14T00:28:32,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/f7f7cdf3ba2c4f54a2ff0112b7031c41 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f7f7cdf3ba2c4f54a2ff0112b7031c41 2024-11-14T00:28:32,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f7f7cdf3ba2c4f54a2ff0112b7031c41, entries=150, sequenceid=52, filesize=11.7 K 2024-11-14T00:28:32,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/21b9a8f68dbd42fe891bdd0dae2a3058 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/21b9a8f68dbd42fe891bdd0dae2a3058 2024-11-14T00:28:32,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/21b9a8f68dbd42fe891bdd0dae2a3058, entries=150, sequenceid=52, filesize=11.7 K 2024-11-14T00:28:32,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 328b146c51a69a897800000a78ffe3a7 in 641ms, sequenceid=52, compaction requested=true 2024-11-14T00:28:32,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:32,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:32,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:32,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:32,492 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:32,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:32,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:32,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:28:32,492 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:32,499 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:32,499 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:32,499 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/B is initiating minor compaction (all files) 2024-11-14T00:28:32,499 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/B in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:32,499 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/9a38c8b8998349f7b3f54fac21190878, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/1606cd57cfc24183a3e37c302002947b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f7f7cdf3ba2c4f54a2ff0112b7031c41] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=35.2 K 2024-11-14T00:28:32,504 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/A is initiating minor compaction (all files) 2024-11-14T00:28:32,504 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/A in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:32,504 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a38c8b8998349f7b3f54fac21190878, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731544109949 2024-11-14T00:28:32,504 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/00e2c348637e4272856f71dc8bc3cf2d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/7e9f9ab83d3c41aeb3320c9cd0ae9bb9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/111994165bf64b7b84bc98007f83ea41] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=37.4 K 2024-11-14T00:28:32,508 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00e2c348637e4272856f71dc8bc3cf2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731544109949 2024-11-14T00:28:32,509 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 1606cd57cfc24183a3e37c302002947b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731544110014 2024-11-14T00:28:32,514 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e9f9ab83d3c41aeb3320c9cd0ae9bb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731544110014 2024-11-14T00:28:32,514 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting f7f7cdf3ba2c4f54a2ff0112b7031c41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544111194 2024-11-14T00:28:32,515 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 111994165bf64b7b84bc98007f83ea41, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544111194 2024-11-14T00:28:32,555 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#B#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:32,556 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#A#compaction#172 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:32,556 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/52cc026e0c3040bbaec7aaae4797dd01 is 50, key is test_row_0/B:col10/1731544111220/Put/seqid=0 2024-11-14T00:28:32,559 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/23de917087e3429e87ce241948844434 is 50, key is test_row_0/A:col10/1731544111220/Put/seqid=0 2024-11-14T00:28:32,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-14T00:28:32,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:32,585 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T00:28:32,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742038_1214 (size=12104) 2024-11-14T00:28:32,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:32,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:32,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742039_1215 (size=12104) 2024-11-14T00:28:32,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/d215ea7ca0d54556a8e0fcb96ab4e6f4 is 50, key is test_row_0/A:col10/1731544112579/Put/seqid=0 2024-11-14T00:28:32,635 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/23de917087e3429e87ce241948844434 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/23de917087e3429e87ce241948844434 2024-11-14T00:28:32,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742040_1216 (size=12001) 2024-11-14T00:28:32,653 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/A of 328b146c51a69a897800000a78ffe3a7 into 23de917087e3429e87ce241948844434(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:32,653 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:32,653 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/A, priority=13, startTime=1731544112491; duration=0sec 2024-11-14T00:28:32,653 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:32,653 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:A 2024-11-14T00:28:32,653 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:32,655 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:32,655 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/C is initiating minor compaction (all files) 2024-11-14T00:28:32,655 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/C in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:32,656 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e83950b6268b4baf8d22c74035583a72, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/53c275da8dc749d8b97888754923af5b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/21b9a8f68dbd42fe891bdd0dae2a3058] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=35.2 K 2024-11-14T00:28:32,657 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting e83950b6268b4baf8d22c74035583a72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731544109949 2024-11-14T00:28:32,657 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53c275da8dc749d8b97888754923af5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731544110014 2024-11-14T00:28:32,658 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21b9a8f68dbd42fe891bdd0dae2a3058, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544111194 2024-11-14T00:28:32,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544172655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544172658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544172667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544172669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544172675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,698 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#C#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:32,698 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/6c7c1e9a664445c9b8367261aa4123e9 is 50, key is test_row_0/C:col10/1731544111220/Put/seqid=0 2024-11-14T00:28:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T00:28:32,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742041_1217 (size=12104) 2024-11-14T00:28:32,765 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/6c7c1e9a664445c9b8367261aa4123e9 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/6c7c1e9a664445c9b8367261aa4123e9 2024-11-14T00:28:32,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544172776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544172776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,785 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/C of 328b146c51a69a897800000a78ffe3a7 into 6c7c1e9a664445c9b8367261aa4123e9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:32,785 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:32,786 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/C, priority=13, startTime=1731544112492; duration=0sec 2024-11-14T00:28:32,786 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:32,786 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:C 2024-11-14T00:28:32,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544172786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544172786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544172786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,862 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T00:28:32,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544172986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:32,998 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/52cc026e0c3040bbaec7aaae4797dd01 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/52cc026e0c3040bbaec7aaae4797dd01 2024-11-14T00:28:32,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544172995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:32,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544172995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544172996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544172996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,011 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/B of 328b146c51a69a897800000a78ffe3a7 into 52cc026e0c3040bbaec7aaae4797dd01(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:33,011 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:33,011 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/B, priority=13, startTime=1731544112492; duration=0sec 2024-11-14T00:28:33,011 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:33,011 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:B 2024-11-14T00:28:33,040 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/d215ea7ca0d54556a8e0fcb96ab4e6f4 2024-11-14T00:28:33,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/ce3153ec72cc42eaadc5a30fda5658c8 is 50, key is test_row_0/B:col10/1731544112579/Put/seqid=0 2024-11-14T00:28:33,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742042_1218 (size=12001) 2024-11-14T00:28:33,104 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/ce3153ec72cc42eaadc5a30fda5658c8 2024-11-14T00:28:33,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/9c04391a7c134bb6b7094f154c1d230c is 50, key is test_row_0/C:col10/1731544112579/Put/seqid=0 2024-11-14T00:28:33,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742043_1219 (size=12001) 2024-11-14T00:28:33,149 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/9c04391a7c134bb6b7094f154c1d230c 2024-11-14T00:28:33,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/d215ea7ca0d54556a8e0fcb96ab4e6f4 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d215ea7ca0d54556a8e0fcb96ab4e6f4 2024-11-14T00:28:33,192 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d215ea7ca0d54556a8e0fcb96ab4e6f4, entries=150, sequenceid=74, filesize=11.7 K 2024-11-14T00:28:33,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/ce3153ec72cc42eaadc5a30fda5658c8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/ce3153ec72cc42eaadc5a30fda5658c8 2024-11-14T00:28:33,201 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/ce3153ec72cc42eaadc5a30fda5658c8, entries=150, sequenceid=74, filesize=11.7 K 2024-11-14T00:28:33,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/9c04391a7c134bb6b7094f154c1d230c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/9c04391a7c134bb6b7094f154c1d230c 2024-11-14T00:28:33,212 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/9c04391a7c134bb6b7094f154c1d230c, entries=150, sequenceid=74, filesize=11.7 K 2024-11-14T00:28:33,214 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 328b146c51a69a897800000a78ffe3a7 in 630ms, sequenceid=74, compaction requested=false 2024-11-14T00:28:33,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:33,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:33,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-14T00:28:33,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-14T00:28:33,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-14T00:28:33,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1140 sec 2024-11-14T00:28:33,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.1300 sec 2024-11-14T00:28:33,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T00:28:33,235 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:33,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-14T00:28:33,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T00:28:33,241 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:33,242 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:33,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:33,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:33,298 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:33,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T00:28:33,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/c502a522fea4455cb9c59cf4010e0180 is 50, key is test_row_0/A:col10/1731544112641/Put/seqid=0 2024-11-14T00:28:33,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742044_1220 (size=14341) 2024-11-14T00:28:33,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/c502a522fea4455cb9c59cf4010e0180 2024-11-14T00:28:33,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544173329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544173329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544173336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544173336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T00:28:33,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544173342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/2eae2ae581ff4d13bbf4605b6ececa6c is 50, key is test_row_0/B:col10/1731544112641/Put/seqid=0 2024-11-14T00:28:33,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742045_1221 (size=12001) 2024-11-14T00:28:33,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T00:28:33,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:33,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:33,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:33,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:33,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:33,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/2eae2ae581ff4d13bbf4605b6ececa6c 2024-11-14T00:28:33,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544173449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544173450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544173455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544173455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/f4eae73f7aa746aab1ef4b22a537d004 is 50, key is test_row_0/C:col10/1731544112641/Put/seqid=0 2024-11-14T00:28:33,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544173458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742046_1222 (size=12001) 2024-11-14T00:28:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T00:28:33,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T00:28:33,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:33,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:33,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:33,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:33,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:33,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544173656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544173667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544173667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544173668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544173676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T00:28:33,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:33,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:33,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:33,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:33,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T00:28:33,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T00:28:33,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:33,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:33,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:33,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:33,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:33,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/f4eae73f7aa746aab1ef4b22a537d004 2024-11-14T00:28:33,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/c502a522fea4455cb9c59cf4010e0180 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/c502a522fea4455cb9c59cf4010e0180 2024-11-14T00:28:33,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/c502a522fea4455cb9c59cf4010e0180, entries=200, sequenceid=92, filesize=14.0 K 2024-11-14T00:28:33,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/2eae2ae581ff4d13bbf4605b6ececa6c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/2eae2ae581ff4d13bbf4605b6ececa6c 2024-11-14T00:28:33,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/2eae2ae581ff4d13bbf4605b6ececa6c, entries=150, sequenceid=92, filesize=11.7 K 2024-11-14T00:28:33,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/f4eae73f7aa746aab1ef4b22a537d004 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/f4eae73f7aa746aab1ef4b22a537d004 2024-11-14T00:28:33,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:33,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/f4eae73f7aa746aab1ef4b22a537d004, entries=150, sequenceid=92, filesize=11.7 K 2024-11-14T00:28:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544173972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:33,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 328b146c51a69a897800000a78ffe3a7 in 686ms, sequenceid=92, compaction requested=true 2024-11-14T00:28:33,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:33,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:33,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:33,985 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:33,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:33,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:33,985 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:33,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:33,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:33,986 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:33,986 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/B is initiating minor compaction (all files) 2024-11-14T00:28:33,986 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/B in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:33,986 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/52cc026e0c3040bbaec7aaae4797dd01, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/ce3153ec72cc42eaadc5a30fda5658c8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/2eae2ae581ff4d13bbf4605b6ececa6c] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=35.3 K 2024-11-14T00:28:33,987 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:33,987 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/A is initiating minor compaction (all files) 2024-11-14T00:28:33,987 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/A in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:33,987 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/23de917087e3429e87ce241948844434, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d215ea7ca0d54556a8e0fcb96ab4e6f4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/c502a522fea4455cb9c59cf4010e0180] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=37.5 K 2024-11-14T00:28:33,987 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 52cc026e0c3040bbaec7aaae4797dd01, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544111194 2024-11-14T00:28:33,989 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23de917087e3429e87ce241948844434, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544111194 2024-11-14T00:28:33,989 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting ce3153ec72cc42eaadc5a30fda5658c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731544111896 2024-11-14T00:28:33,990 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting d215ea7ca0d54556a8e0fcb96ab4e6f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731544111896 2024-11-14T00:28:33,991 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting c502a522fea4455cb9c59cf4010e0180, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731544112641 2024-11-14T00:28:33,991 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eae2ae581ff4d13bbf4605b6ececa6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731544112641 2024-11-14T00:28:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:33,993 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:33,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T00:28:34,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/42c5f79443e94362901be2a06cefae4f is 50, key is test_row_0/A:col10/1731544113989/Put/seqid=0 2024-11-14T00:28:34,019 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#B#compaction#181 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:34,019 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/49f23a428adf4cc08728bfae57e3da27 is 50, key is test_row_0/B:col10/1731544112641/Put/seqid=0 2024-11-14T00:28:34,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544174013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544174018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544174019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544174021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,030 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#A#compaction#182 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:34,031 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/ddd24d6a42ca446baa1976fdf8842732 is 50, key is test_row_0/A:col10/1731544112641/Put/seqid=0 2024-11-14T00:28:34,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T00:28:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:34,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742047_1223 (size=12001) 2024-11-14T00:28:34,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/42c5f79443e94362901be2a06cefae4f 2024-11-14T00:28:34,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742048_1224 (size=12207) 2024-11-14T00:28:34,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/8626967a3af64623832bff64143199da is 50, key is test_row_0/B:col10/1731544113989/Put/seqid=0 2024-11-14T00:28:34,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742049_1225 (size=12207) 2024-11-14T00:28:34,121 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/49f23a428adf4cc08728bfae57e3da27 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/49f23a428adf4cc08728bfae57e3da27 2024-11-14T00:28:34,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742050_1226 (size=12001) 2024-11-14T00:28:34,131 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/B of 328b146c51a69a897800000a78ffe3a7 into 49f23a428adf4cc08728bfae57e3da27(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:34,132 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:34,132 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/B, priority=13, startTime=1731544113985; duration=0sec 2024-11-14T00:28:34,132 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:34,132 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:B 2024-11-14T00:28:34,132 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:34,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544174129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/8626967a3af64623832bff64143199da 2024-11-14T00:28:34,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544174135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544174135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544174138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,148 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:34,148 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/C is initiating minor compaction (all files) 2024-11-14T00:28:34,149 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/C in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:34,149 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/6c7c1e9a664445c9b8367261aa4123e9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/9c04391a7c134bb6b7094f154c1d230c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/f4eae73f7aa746aab1ef4b22a537d004] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=35.3 K 2024-11-14T00:28:34,150 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c7c1e9a664445c9b8367261aa4123e9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544111194 2024-11-14T00:28:34,150 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c04391a7c134bb6b7094f154c1d230c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731544111896 2024-11-14T00:28:34,151 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting f4eae73f7aa746aab1ef4b22a537d004, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731544112641 2024-11-14T00:28:34,167 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#C#compaction#184 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:34,168 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/322171c2b4a043798ef9e01588947a39 is 50, key is test_row_0/C:col10/1731544112641/Put/seqid=0 2024-11-14T00:28:34,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T00:28:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:34,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:34,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:34,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/e6d145db5fa640b2b462e8c20e374298 is 50, key is test_row_0/C:col10/1731544113989/Put/seqid=0 2024-11-14T00:28:34,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742051_1227 (size=12207) 2024-11-14T00:28:34,247 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/322171c2b4a043798ef9e01588947a39 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/322171c2b4a043798ef9e01588947a39 2024-11-14T00:28:34,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742052_1228 (size=12001) 2024-11-14T00:28:34,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/e6d145db5fa640b2b462e8c20e374298 2024-11-14T00:28:34,262 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/C of 328b146c51a69a897800000a78ffe3a7 into 322171c2b4a043798ef9e01588947a39(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:34,262 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:34,262 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/C, priority=13, startTime=1731544113985; duration=0sec 2024-11-14T00:28:34,262 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:34,262 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:C 2024-11-14T00:28:34,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/42c5f79443e94362901be2a06cefae4f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/42c5f79443e94362901be2a06cefae4f 2024-11-14T00:28:34,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/42c5f79443e94362901be2a06cefae4f, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T00:28:34,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/8626967a3af64623832bff64143199da as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/8626967a3af64623832bff64143199da 2024-11-14T00:28:34,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/8626967a3af64623832bff64143199da, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T00:28:34,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/e6d145db5fa640b2b462e8c20e374298 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e6d145db5fa640b2b462e8c20e374298 2024-11-14T00:28:34,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T00:28:34,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:34,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:34,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:34,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e6d145db5fa640b2b462e8c20e374298, entries=150, sequenceid=116, filesize=11.7 K 2024-11-14T00:28:34,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 328b146c51a69a897800000a78ffe3a7 in 360ms, sequenceid=116, compaction requested=false 2024-11-14T00:28:34,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:34,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:34,360 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:34,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T00:28:34,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/c896e068548d4b7a85c5fabcec94c08f is 50, key is test_row_0/A:col10/1731544114016/Put/seqid=0 2024-11-14T00:28:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T00:28:34,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742053_1229 (size=12051) 2024-11-14T00:28:34,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544174436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544174441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544174441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544174442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544174495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T00:28:34,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:34,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:34,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:34,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,530 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/ddd24d6a42ca446baa1976fdf8842732 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/ddd24d6a42ca446baa1976fdf8842732 2024-11-14T00:28:34,548 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/A of 328b146c51a69a897800000a78ffe3a7 into ddd24d6a42ca446baa1976fdf8842732(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:34,548 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:34,548 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/A, priority=13, startTime=1731544113984; duration=0sec 2024-11-14T00:28:34,548 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:34,548 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:A 2024-11-14T00:28:34,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544174547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544174555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544174555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544174588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T00:28:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:34,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544174765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544174765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544174768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/c896e068548d4b7a85c5fabcec94c08f 2024-11-14T00:28:34,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544174795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:34,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:34,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/01d9cfaba4b74466bafda523025a66bc is 50, key is test_row_0/B:col10/1731544114016/Put/seqid=0 2024-11-14T00:28:34,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T00:28:34,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:34,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:34,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:34,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742054_1230 (size=12051) 2024-11-14T00:28:34,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T00:28:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:34,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:35,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544175076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:35,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544175076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:35,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544175088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:35,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544175105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:35,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T00:28:35,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:35,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:35,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:35,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:35,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:35,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:35,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/01d9cfaba4b74466bafda523025a66bc 2024-11-14T00:28:35,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/217c1d40585d4abaa06f1fdf563cd7c1 is 50, key is test_row_0/C:col10/1731544114016/Put/seqid=0 2024-11-14T00:28:35,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742055_1231 (size=12051) 2024-11-14T00:28:35,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T00:28:35,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:35,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:35,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:35,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:35,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:35,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T00:28:35,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T00:28:35,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:35,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:35,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:35,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:35,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:35,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:35,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544175506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:35,512 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_0' in 328b146c51a69a897800000a78ffe3a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:28:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:35,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544175585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:35,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:35,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544175588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:35,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:35,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544175606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:35,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544175626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:35,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T00:28:35,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:35,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:35,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:35,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:35,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:35,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/217c1d40585d4abaa06f1fdf563cd7c1 2024-11-14T00:28:35,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/c896e068548d4b7a85c5fabcec94c08f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/c896e068548d4b7a85c5fabcec94c08f 2024-11-14T00:28:35,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/c896e068548d4b7a85c5fabcec94c08f, entries=150, sequenceid=131, filesize=11.8 K 2024-11-14T00:28:35,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/01d9cfaba4b74466bafda523025a66bc as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/01d9cfaba4b74466bafda523025a66bc 2024-11-14T00:28:35,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/01d9cfaba4b74466bafda523025a66bc, entries=150, sequenceid=131, filesize=11.8 K 2024-11-14T00:28:35,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/217c1d40585d4abaa06f1fdf563cd7c1 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/217c1d40585d4abaa06f1fdf563cd7c1 2024-11-14T00:28:35,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/217c1d40585d4abaa06f1fdf563cd7c1, entries=150, sequenceid=131, filesize=11.8 K 2024-11-14T00:28:35,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 328b146c51a69a897800000a78ffe3a7 in 1416ms, sequenceid=131, compaction requested=true 2024-11-14T00:28:35,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:35,776 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:35,778 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:35,778 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/A is initiating minor compaction (all files) 2024-11-14T00:28:35,778 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/A in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:35,778 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/ddd24d6a42ca446baa1976fdf8842732, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/42c5f79443e94362901be2a06cefae4f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/c896e068548d4b7a85c5fabcec94c08f] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=35.4 K 2024-11-14T00:28:35,778 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddd24d6a42ca446baa1976fdf8842732, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731544112641 2024-11-14T00:28:35,779 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42c5f79443e94362901be2a06cefae4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731544113334 2024-11-14T00:28:35,779 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting c896e068548d4b7a85c5fabcec94c08f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731544114016 2024-11-14T00:28:35,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:35,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:35,785 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:35,786 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:35,786 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/B is initiating minor compaction (all files) 2024-11-14T00:28:35,786 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/B in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:35,786 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/49f23a428adf4cc08728bfae57e3da27, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/8626967a3af64623832bff64143199da, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/01d9cfaba4b74466bafda523025a66bc] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=35.4 K 2024-11-14T00:28:35,787 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 49f23a428adf4cc08728bfae57e3da27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731544112641 2024-11-14T00:28:35,787 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 8626967a3af64623832bff64143199da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731544113334 2024-11-14T00:28:35,787 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 01d9cfaba4b74466bafda523025a66bc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731544114016 2024-11-14T00:28:35,791 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#A#compaction#189 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:35,792 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/ebddf344a8f347be9d346029c991c05b is 50, key is test_row_0/A:col10/1731544114016/Put/seqid=0 2024-11-14T00:28:35,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:35,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:35,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:35,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:35,803 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#B#compaction#190 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:35,803 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/01cfb3e56cce4a00b67128fa6eb3a3f9 is 50, key is test_row_0/B:col10/1731544114016/Put/seqid=0 2024-11-14T00:28:35,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-14T00:28:35,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:35,813 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T00:28:35,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742057_1233 (size=12359) 2024-11-14T00:28:35,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/a8685dd4836a425887225de5269b0234 is 50, key is test_row_0/A:col10/1731544114429/Put/seqid=0 2024-11-14T00:28:35,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742056_1232 (size=12359) 2024-11-14T00:28:35,863 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/01cfb3e56cce4a00b67128fa6eb3a3f9 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/01cfb3e56cce4a00b67128fa6eb3a3f9 2024-11-14T00:28:35,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742058_1234 (size=12151) 2024-11-14T00:28:35,876 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/B of 328b146c51a69a897800000a78ffe3a7 into 01cfb3e56cce4a00b67128fa6eb3a3f9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:35,876 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:35,876 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/B, priority=13, startTime=1731544115785; duration=0sec 2024-11-14T00:28:35,877 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:35,877 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:B 2024-11-14T00:28:35,877 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:35,881 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:35,881 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/C is initiating minor compaction (all files) 2024-11-14T00:28:35,881 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/C in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:35,881 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/322171c2b4a043798ef9e01588947a39, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e6d145db5fa640b2b462e8c20e374298, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/217c1d40585d4abaa06f1fdf563cd7c1] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=35.4 K 2024-11-14T00:28:35,882 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 322171c2b4a043798ef9e01588947a39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731544112641 2024-11-14T00:28:35,883 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting e6d145db5fa640b2b462e8c20e374298, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731544113334 2024-11-14T00:28:35,884 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 217c1d40585d4abaa06f1fdf563cd7c1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731544114016 2024-11-14T00:28:35,900 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#C#compaction#192 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:35,900 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/8b2fd3f28d5c4a93af51199404766bd6 is 50, key is test_row_0/C:col10/1731544114016/Put/seqid=0 2024-11-14T00:28:35,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742059_1235 (size=12359) 2024-11-14T00:28:36,277 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/a8685dd4836a425887225de5269b0234 2024-11-14T00:28:36,298 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/ebddf344a8f347be9d346029c991c05b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/ebddf344a8f347be9d346029c991c05b 2024-11-14T00:28:36,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/3f47874e343e49639c7fc4898831a2a0 is 50, key is test_row_0/B:col10/1731544114429/Put/seqid=0 2024-11-14T00:28:36,324 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/A of 328b146c51a69a897800000a78ffe3a7 into ebddf344a8f347be9d346029c991c05b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:36,324 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:36,324 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/A, priority=13, startTime=1731544115776; duration=0sec 2024-11-14T00:28:36,324 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:36,324 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:A 2024-11-14T00:28:36,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742060_1236 (size=12151) 2024-11-14T00:28:36,369 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/3f47874e343e49639c7fc4898831a2a0 2024-11-14T00:28:36,389 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/8b2fd3f28d5c4a93af51199404766bd6 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/8b2fd3f28d5c4a93af51199404766bd6 2024-11-14T00:28:36,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/a33554ed234c42e7ac62f71a664ec6e3 is 50, key is test_row_0/C:col10/1731544114429/Put/seqid=0 2024-11-14T00:28:36,423 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/C of 328b146c51a69a897800000a78ffe3a7 into 8b2fd3f28d5c4a93af51199404766bd6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:36,423 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:36,423 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/C, priority=13, startTime=1731544115798; duration=0sec 2024-11-14T00:28:36,423 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:36,423 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:C 2024-11-14T00:28:36,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742061_1237 (size=12151) 2024-11-14T00:28:36,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:36,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544176622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:36,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:36,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544176626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:36,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544176640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:36,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544176645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:36,648 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_0' in 328b146c51a69a897800000a78ffe3a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:28:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:36,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544176735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:36,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544176736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:36,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544176746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:36,860 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/a33554ed234c42e7ac62f71a664ec6e3 2024-11-14T00:28:36,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/a8685dd4836a425887225de5269b0234 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a8685dd4836a425887225de5269b0234 2024-11-14T00:28:36,925 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a8685dd4836a425887225de5269b0234, entries=150, sequenceid=157, filesize=11.9 K 2024-11-14T00:28:36,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/3f47874e343e49639c7fc4898831a2a0 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/3f47874e343e49639c7fc4898831a2a0 2024-11-14T00:28:36,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:36,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544176945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:36,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544176945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:36,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:36,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:36,951 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/3f47874e343e49639c7fc4898831a2a0, entries=150, sequenceid=157, filesize=11.9 K 2024-11-14T00:28:36,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/a33554ed234c42e7ac62f71a664ec6e3 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/a33554ed234c42e7ac62f71a664ec6e3 2024-11-14T00:28:36,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:36,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544176956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:36,974 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/a33554ed234c42e7ac62f71a664ec6e3, entries=150, sequenceid=157, filesize=11.9 K 2024-11-14T00:28:36,976 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 328b146c51a69a897800000a78ffe3a7 in 1164ms, sequenceid=157, compaction requested=false 2024-11-14T00:28:36,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:36,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:36,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-14T00:28:36,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-14T00:28:36,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-14T00:28:36,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7440 sec 2024-11-14T00:28:36,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 3.7580 sec 2024-11-14T00:28:37,259 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:37,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T00:28:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:37,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/a9d32fb5312448968e63e9a9590e9e45 is 50, key is test_row_0/A:col10/1731544117258/Put/seqid=0 2024-11-14T00:28:37,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742062_1238 (size=16931) 2024-11-14T00:28:37,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/a9d32fb5312448968e63e9a9590e9e45 2024-11-14T00:28:37,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/ac4d756817ea420c8401a4bc78493611 is 50, key is test_row_0/B:col10/1731544117258/Put/seqid=0 2024-11-14T00:28:37,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544177320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:37,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544177324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:37,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544177325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:37,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742063_1239 (size=12151) 2024-11-14T00:28:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-14T00:28:37,408 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:37,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-14T00:28:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T00:28:37,427 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:37,436 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:37,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:37,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544177435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:37,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544177435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:37,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544177440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:37,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:37,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544177528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T00:28:37,536 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_0' in 328b146c51a69a897800000a78ffe3a7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:28:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:37,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T00:28:37,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:37,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:37,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:37,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:37,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:37,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:37,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544177648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544177648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:37,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:37,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544177656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:37,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T00:28:37,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/ac4d756817ea420c8401a4bc78493611 2024-11-14T00:28:37,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T00:28:37,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:37,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:37,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:37,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:37,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:37,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:37,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/148b7ceb4cdf4a77bf5ed424b27ae01a is 50, key is test_row_0/C:col10/1731544117258/Put/seqid=0 2024-11-14T00:28:37,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742064_1240 (size=12151) 2024-11-14T00:28:37,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T00:28:37,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:37,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:37,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:37,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:37,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:37,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544177956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:37,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544177964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:37,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:37,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544177968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T00:28:38,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T00:28:38,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:38,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:38,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:38,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T00:28:38,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:38,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:38,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:38,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/148b7ceb4cdf4a77bf5ed424b27ae01a 2024-11-14T00:28:38,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/a9d32fb5312448968e63e9a9590e9e45 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a9d32fb5312448968e63e9a9590e9e45 2024-11-14T00:28:38,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a9d32fb5312448968e63e9a9590e9e45, entries=250, sequenceid=174, filesize=16.5 K 2024-11-14T00:28:38,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/ac4d756817ea420c8401a4bc78493611 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/ac4d756817ea420c8401a4bc78493611 2024-11-14T00:28:38,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/ac4d756817ea420c8401a4bc78493611, entries=150, sequenceid=174, filesize=11.9 K 2024-11-14T00:28:38,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/148b7ceb4cdf4a77bf5ed424b27ae01a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/148b7ceb4cdf4a77bf5ed424b27ae01a 2024-11-14T00:28:38,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T00:28:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:38,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/148b7ceb4cdf4a77bf5ed424b27ae01a, entries=150, sequenceid=174, filesize=11.9 K 2024-11-14T00:28:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 328b146c51a69a897800000a78ffe3a7 in 1173ms, sequenceid=174, compaction requested=true 2024-11-14T00:28:38,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:38,433 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:38,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:38,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:38,434 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:38,435 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:38,435 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/A is initiating minor compaction (all files) 2024-11-14T00:28:38,435 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/A in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:38,435 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/ebddf344a8f347be9d346029c991c05b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a8685dd4836a425887225de5269b0234, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a9d32fb5312448968e63e9a9590e9e45] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=40.5 K 2024-11-14T00:28:38,436 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:38,437 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/B is initiating minor compaction (all files) 2024-11-14T00:28:38,437 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/B in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:38,437 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/01cfb3e56cce4a00b67128fa6eb3a3f9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/3f47874e343e49639c7fc4898831a2a0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/ac4d756817ea420c8401a4bc78493611] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=35.8 K 2024-11-14T00:28:38,437 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebddf344a8f347be9d346029c991c05b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731544114016 2024-11-14T00:28:38,437 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 01cfb3e56cce4a00b67128fa6eb3a3f9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731544114016 2024-11-14T00:28:38,438 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8685dd4836a425887225de5269b0234, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731544114429 2024-11-14T00:28:38,438 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f47874e343e49639c7fc4898831a2a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731544114429 2024-11-14T00:28:38,438 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9d32fb5312448968e63e9a9590e9e45, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731544116606 2024-11-14T00:28:38,439 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting ac4d756817ea420c8401a4bc78493611, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731544116606 2024-11-14T00:28:38,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:38,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:38,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:38,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:38,453 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#B#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:38,454 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/2ad93d65b94940e7be8e4c60707e6bb1 is 50, key is test_row_0/B:col10/1731544117258/Put/seqid=0 2024-11-14T00:28:38,464 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#A#compaction#199 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:38,465 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/bb042bf553cd483abb5cd0c52d35c5e3 is 50, key is test_row_0/A:col10/1731544117258/Put/seqid=0 2024-11-14T00:28:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:38,476 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:38,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T00:28:38,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742066_1242 (size=12561) 2024-11-14T00:28:38,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/e0a7406a80f34fde8f3435985a565595 is 50, key is test_row_0/A:col10/1731544118472/Put/seqid=0 2024-11-14T00:28:38,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742065_1241 (size=12561) 2024-11-14T00:28:38,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:38,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544178506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:38,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544178514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:38,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:38,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544178514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:38,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:38,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:38,530 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/bb042bf553cd483abb5cd0c52d35c5e3 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/bb042bf553cd483abb5cd0c52d35c5e3 2024-11-14T00:28:38,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742067_1243 (size=14541) 2024-11-14T00:28:38,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/e0a7406a80f34fde8f3435985a565595 2024-11-14T00:28:38,561 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/A of 328b146c51a69a897800000a78ffe3a7 into bb042bf553cd483abb5cd0c52d35c5e3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:38,561 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:38,561 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/A, priority=13, startTime=1731544118432; duration=0sec 2024-11-14T00:28:38,562 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:38,562 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:A 2024-11-14T00:28:38,562 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:38,569 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:38,569 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/C is initiating minor compaction (all files) 2024-11-14T00:28:38,569 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/C in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:38,569 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/8b2fd3f28d5c4a93af51199404766bd6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/a33554ed234c42e7ac62f71a664ec6e3, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/148b7ceb4cdf4a77bf5ed424b27ae01a] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=35.8 K 2024-11-14T00:28:38,570 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b2fd3f28d5c4a93af51199404766bd6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731544114016 2024-11-14T00:28:38,570 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting a33554ed234c42e7ac62f71a664ec6e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731544114429 2024-11-14T00:28:38,573 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 148b7ceb4cdf4a77bf5ed424b27ae01a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731544116606 2024-11-14T00:28:38,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T00:28:38,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T00:28:38,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:38,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:38,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:38,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/6fb69cd9e5ab48ddae083c36d51ad4ff is 50, key is test_row_0/B:col10/1731544118472/Put/seqid=0 2024-11-14T00:28:38,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544178626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:38,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:38,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544178628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:38,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:38,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544178628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:38,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,632 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:38,638 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#C#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:38,639 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/545faf0cfce542c0b112fe98963dc30e is 50, key is test_row_0/C:col10/1731544117258/Put/seqid=0 2024-11-14T00:28:38,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742068_1244 (size=12151) 2024-11-14T00:28:38,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:38,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544178665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:38,672 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_0' in 328b146c51a69a897800000a78ffe3a7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:28:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:38,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742069_1245 (size=12561) 2024-11-14T00:28:38,739 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/545faf0cfce542c0b112fe98963dc30e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/545faf0cfce542c0b112fe98963dc30e 2024-11-14T00:28:38,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T00:28:38,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:38,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:38,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:38,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,779 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/C of 328b146c51a69a897800000a78ffe3a7 into 545faf0cfce542c0b112fe98963dc30e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:38,779 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:38,779 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/C, priority=13, startTime=1731544118449; duration=0sec 2024-11-14T00:28:38,779 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:38,779 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:C 2024-11-14T00:28:38,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544178835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:38,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544178835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:38,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:38,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544178835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:38,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:38,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:38,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:38,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T00:28:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:38,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:38,932 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/2ad93d65b94940e7be8e4c60707e6bb1 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/2ad93d65b94940e7be8e4c60707e6bb1 2024-11-14T00:28:38,960 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/B of 328b146c51a69a897800000a78ffe3a7 into 2ad93d65b94940e7be8e4c60707e6bb1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:38,960 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:38,960 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/B, priority=13, startTime=1731544118434; duration=0sec 2024-11-14T00:28:38,961 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:38,961 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:B 2024-11-14T00:28:39,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/6fb69cd9e5ab48ddae083c36d51ad4ff 2024-11-14T00:28:39,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T00:28:39,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:39,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:39,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:39,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:39,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:39,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/7346dc25b0904ec9b4fecf6b09cd2e1d is 50, key is test_row_0/C:col10/1731544118472/Put/seqid=0 2024-11-14T00:28:39,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:39,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742070_1246 (size=12151) 2024-11-14T00:28:39,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/7346dc25b0904ec9b4fecf6b09cd2e1d 2024-11-14T00:28:39,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/e0a7406a80f34fde8f3435985a565595 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/e0a7406a80f34fde8f3435985a565595 2024-11-14T00:28:39,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/e0a7406a80f34fde8f3435985a565595, entries=200, sequenceid=197, filesize=14.2 K 2024-11-14T00:28:39,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/6fb69cd9e5ab48ddae083c36d51ad4ff as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/6fb69cd9e5ab48ddae083c36d51ad4ff 2024-11-14T00:28:39,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/6fb69cd9e5ab48ddae083c36d51ad4ff, entries=150, sequenceid=197, filesize=11.9 K 2024-11-14T00:28:39,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/7346dc25b0904ec9b4fecf6b09cd2e1d as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/7346dc25b0904ec9b4fecf6b09cd2e1d 2024-11-14T00:28:39,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544179145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:39,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544179146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:39,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:39,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:39,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544179148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:39,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/7346dc25b0904ec9b4fecf6b09cd2e1d, entries=150, sequenceid=197, filesize=11.9 K 2024-11-14T00:28:39,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 328b146c51a69a897800000a78ffe3a7 in 683ms, sequenceid=197, compaction requested=false 2024-11-14T00:28:39,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:39,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-14T00:28:39,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:39,239 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T00:28:39,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/737e4132d8594304b45f605a7be04d5e is 50, key is test_row_0/A:col10/1731544118507/Put/seqid=0 2024-11-14T00:28:39,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742071_1247 (size=12151) 2024-11-14T00:28:39,311 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/737e4132d8594304b45f605a7be04d5e 2024-11-14T00:28:39,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/aa0e152083684eb184593a4c53fed5b9 is 50, key is test_row_0/B:col10/1731544118507/Put/seqid=0 2024-11-14T00:28:39,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742072_1248 (size=12151) 2024-11-14T00:28:39,370 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/aa0e152083684eb184593a4c53fed5b9 2024-11-14T00:28:39,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/c72a609fc0e14994b141f30e27cfeef0 is 50, key is test_row_0/C:col10/1731544118507/Put/seqid=0 2024-11-14T00:28:39,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742073_1249 (size=12151) 2024-11-14T00:28:39,455 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/c72a609fc0e14994b141f30e27cfeef0 2024-11-14T00:28:39,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/737e4132d8594304b45f605a7be04d5e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/737e4132d8594304b45f605a7be04d5e 2024-11-14T00:28:39,526 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/737e4132d8594304b45f605a7be04d5e, entries=150, sequenceid=213, filesize=11.9 K 2024-11-14T00:28:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/aa0e152083684eb184593a4c53fed5b9 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/aa0e152083684eb184593a4c53fed5b9 2024-11-14T00:28:39,545 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/aa0e152083684eb184593a4c53fed5b9, entries=150, sequenceid=213, filesize=11.9 K 2024-11-14T00:28:39,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/c72a609fc0e14994b141f30e27cfeef0 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c72a609fc0e14994b141f30e27cfeef0 2024-11-14T00:28:39,557 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c72a609fc0e14994b141f30e27cfeef0, entries=150, sequenceid=213, filesize=11.9 K 2024-11-14T00:28:39,560 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=0 B/0 for 328b146c51a69a897800000a78ffe3a7 in 321ms, sequenceid=213, compaction requested=true 2024-11-14T00:28:39,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:39,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:39,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-14T00:28:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-14T00:28:39,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-14T00:28:39,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1390 sec 2024-11-14T00:28:39,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.1700 sec 2024-11-14T00:28:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T00:28:39,598 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:39,599 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-14T00:28:39,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T00:28:39,607 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:39,608 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:39,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:39,692 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:39,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T00:28:39,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/46422312941040648cbf369a2ee24bee is 50, key is test_row_0/A:col10/1731544119686/Put/seqid=0 2024-11-14T00:28:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T00:28:39,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T00:28:39,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742074_1250 (size=12151) 2024-11-14T00:28:39,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:39,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:39,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:39,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:39,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:39,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544179820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:39,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544179825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:39,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:39,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544179837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:39,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:39,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T00:28:39,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:39,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:39,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:39,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:39,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T00:28:39,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544179946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:39,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544179946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:39,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:39,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544179956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:39,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:39,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:40,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T00:28:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:40,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/46422312941040648cbf369a2ee24bee 2024-11-14T00:28:40,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544180162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:40,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544180162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:40,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544180167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:40,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:40,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/d4c49b76d7c34c678c70add0098ec995 is 50, key is test_row_0/B:col10/1731544119686/Put/seqid=0 2024-11-14T00:28:40,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742075_1251 (size=12151) 2024-11-14T00:28:40,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/d4c49b76d7c34c678c70add0098ec995 2024-11-14T00:28:40,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/5dcf7bd4c43f4dbeba6a9c21199eaf2e is 50, key is test_row_0/C:col10/1731544119686/Put/seqid=0 2024-11-14T00:28:40,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T00:28:40,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:40,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:40,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:40,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T00:28:40,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742076_1252 (size=12151) 2024-11-14T00:28:40,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/5dcf7bd4c43f4dbeba6a9c21199eaf2e 2024-11-14T00:28:40,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/46422312941040648cbf369a2ee24bee as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/46422312941040648cbf369a2ee24bee 2024-11-14T00:28:40,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/46422312941040648cbf369a2ee24bee, entries=150, sequenceid=224, filesize=11.9 K 2024-11-14T00:28:40,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/d4c49b76d7c34c678c70add0098ec995 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d4c49b76d7c34c678c70add0098ec995 2024-11-14T00:28:40,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T00:28:40,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d4c49b76d7c34c678c70add0098ec995, entries=150, sequenceid=224, filesize=11.9 K 2024-11-14T00:28:40,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:40,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:40,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:40,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/5dcf7bd4c43f4dbeba6a9c21199eaf2e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/5dcf7bd4c43f4dbeba6a9c21199eaf2e 2024-11-14T00:28:40,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/5dcf7bd4c43f4dbeba6a9c21199eaf2e, entries=150, sequenceid=224, filesize=11.9 K 2024-11-14T00:28:40,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 328b146c51a69a897800000a78ffe3a7 in 757ms, sequenceid=224, compaction requested=true 2024-11-14T00:28:40,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:40,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:40,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:40,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:40,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:28:40,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:40,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T00:28:40,452 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:28:40,456 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:28:40,464 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51404 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:28:40,465 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/A is initiating minor compaction (all files) 2024-11-14T00:28:40,465 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/A in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:40,465 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/bb042bf553cd483abb5cd0c52d35c5e3, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/e0a7406a80f34fde8f3435985a565595, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/737e4132d8594304b45f605a7be04d5e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/46422312941040648cbf369a2ee24bee] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=50.2 K 2024-11-14T00:28:40,465 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb042bf553cd483abb5cd0c52d35c5e3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731544116606 2024-11-14T00:28:40,472 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0a7406a80f34fde8f3435985a565595, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731544117308 2024-11-14T00:28:40,476 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 737e4132d8594304b45f605a7be04d5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731544118499 2024-11-14T00:28:40,476 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:28:40,477 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/C is initiating minor compaction (all files) 2024-11-14T00:28:40,477 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/C in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:40,477 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/545faf0cfce542c0b112fe98963dc30e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/7346dc25b0904ec9b4fecf6b09cd2e1d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c72a609fc0e14994b141f30e27cfeef0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/5dcf7bd4c43f4dbeba6a9c21199eaf2e] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=47.9 K 2024-11-14T00:28:40,478 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46422312941040648cbf369a2ee24bee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731544119675 2024-11-14T00:28:40,479 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 545faf0cfce542c0b112fe98963dc30e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731544116606 2024-11-14T00:28:40,479 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 7346dc25b0904ec9b4fecf6b09cd2e1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731544117314 2024-11-14T00:28:40,483 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c72a609fc0e14994b141f30e27cfeef0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731544118499 2024-11-14T00:28:40,489 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dcf7bd4c43f4dbeba6a9c21199eaf2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731544119675 2024-11-14T00:28:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:40,499 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:40,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-14T00:28:40,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/f93335985b1c43c08241fe803f23543a is 50, key is test_row_0/A:col10/1731544120478/Put/seqid=0 2024-11-14T00:28:40,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:40,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544180521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:40,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:40,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544180537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:40,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544180537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:40,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:40,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:40,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742077_1253 (size=12151) 2024-11-14T00:28:40,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T00:28:40,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:40,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:40,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:40,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,571 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#A#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:40,571 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/92cf30576118492f973ff11e2d0af48e is 50, key is test_row_0/A:col10/1731544119686/Put/seqid=0 2024-11-14T00:28:40,587 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#C#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:40,587 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/dabe352cb1bc41c4b4b5b5659aa47f09 is 50, key is test_row_0/C:col10/1731544119686/Put/seqid=0 2024-11-14T00:28:40,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742078_1254 (size=12697) 2024-11-14T00:28:40,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:40,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544180645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:40,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:40,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742079_1255 (size=12697) 2024-11-14T00:28:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544180648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:40,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544180648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:40,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:40,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T00:28:40,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:40,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:40,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:40,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T00:28:40,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544180856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:40,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:40,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544180867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:40,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544180868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:40,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T00:28:40,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:40,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:40,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:40,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:40,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/f93335985b1c43c08241fe803f23543a 2024-11-14T00:28:40,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/d8128998796541dabde32176ddc20508 is 50, key is test_row_0/B:col10/1731544120478/Put/seqid=0 2024-11-14T00:28:41,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742080_1256 (size=12151) 2024-11-14T00:28:41,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T00:28:41,038 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/92cf30576118492f973ff11e2d0af48e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/92cf30576118492f973ff11e2d0af48e 2024-11-14T00:28:41,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:41,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:41,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:41,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:41,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:41,045 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/A of 328b146c51a69a897800000a78ffe3a7 into 92cf30576118492f973ff11e2d0af48e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:41,045 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:41,045 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/A, priority=12, startTime=1731544120449; duration=0sec 2024-11-14T00:28:41,045 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:41,045 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:A 2024-11-14T00:28:41,045 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:28:41,048 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:28:41,048 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/B is initiating minor compaction (all files) 2024-11-14T00:28:41,048 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/B in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:41,048 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/2ad93d65b94940e7be8e4c60707e6bb1, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/6fb69cd9e5ab48ddae083c36d51ad4ff, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/aa0e152083684eb184593a4c53fed5b9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d4c49b76d7c34c678c70add0098ec995] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=47.9 K 2024-11-14T00:28:41,049 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ad93d65b94940e7be8e4c60707e6bb1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731544116606 2024-11-14T00:28:41,049 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fb69cd9e5ab48ddae083c36d51ad4ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731544117314 2024-11-14T00:28:41,050 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa0e152083684eb184593a4c53fed5b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731544118499 2024-11-14T00:28:41,050 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4c49b76d7c34c678c70add0098ec995, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731544119675 2024-11-14T00:28:41,068 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/dabe352cb1bc41c4b4b5b5659aa47f09 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/dabe352cb1bc41c4b4b5b5659aa47f09 2024-11-14T00:28:41,071 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:41,071 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/d6ede8630e9b49a5a410279329894dfd is 50, key is test_row_0/B:col10/1731544119686/Put/seqid=0 2024-11-14T00:28:41,085 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/C of 328b146c51a69a897800000a78ffe3a7 into dabe352cb1bc41c4b4b5b5659aa47f09(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:41,085 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:41,085 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/C, priority=12, startTime=1731544120450; duration=0sec 2024-11-14T00:28:41,085 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:41,085 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:C 2024-11-14T00:28:41,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742081_1257 (size=12697) 2024-11-14T00:28:41,124 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/d6ede8630e9b49a5a410279329894dfd as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d6ede8630e9b49a5a410279329894dfd 2024-11-14T00:28:41,141 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/B of 328b146c51a69a897800000a78ffe3a7 into d6ede8630e9b49a5a410279329894dfd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:41,141 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:41,141 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/B, priority=12, startTime=1731544120449; duration=0sec 2024-11-14T00:28:41,141 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:41,141 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:B 2024-11-14T00:28:41,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544181175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:41,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544181177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:41,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:41,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:41,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544181192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:41,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T00:28:41,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:41,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:41,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:41,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:41,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:41,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T00:28:41,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:41,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:41,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:41,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:41,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:41,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/d8128998796541dabde32176ddc20508 2024-11-14T00:28:41,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/1c1416405ab649c3a7c522eaa3ac18a7 is 50, key is test_row_0/C:col10/1731544120478/Put/seqid=0 2024-11-14T00:28:41,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742082_1258 (size=12151) 2024-11-14T00:28:41,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/1c1416405ab649c3a7c522eaa3ac18a7 2024-11-14T00:28:41,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/f93335985b1c43c08241fe803f23543a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/f93335985b1c43c08241fe803f23543a 2024-11-14T00:28:41,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T00:28:41,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:41,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:41,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:41,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:41,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:41,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/f93335985b1c43c08241fe803f23543a, entries=150, sequenceid=252, filesize=11.9 K 2024-11-14T00:28:41,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/d8128998796541dabde32176ddc20508 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d8128998796541dabde32176ddc20508 2024-11-14T00:28:41,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d8128998796541dabde32176ddc20508, entries=150, sequenceid=252, filesize=11.9 K 2024-11-14T00:28:41,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/1c1416405ab649c3a7c522eaa3ac18a7 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/1c1416405ab649c3a7c522eaa3ac18a7 2024-11-14T00:28:41,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51586 deadline: 1731544181545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:41,563 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_0' in 328b146c51a69a897800000a78ffe3a7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:28:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:41,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/1c1416405ab649c3a7c522eaa3ac18a7, entries=150, sequenceid=252, filesize=11.9 K 2024-11-14T00:28:41,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 328b146c51a69a897800000a78ffe3a7 in 1085ms, sequenceid=252, compaction requested=false 2024-11-14T00:28:41,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:41,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-14T00:28:41,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:41,693 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-14T00:28:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:41,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:41,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/741203e72717487c98eb64afb2530ba5 is 50, key is test_row_0/A:col10/1731544121687/Put/seqid=0 2024-11-14T00:28:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T00:28:41,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742083_1259 (size=12301) 2024-11-14T00:28:41,784 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/741203e72717487c98eb64afb2530ba5 2024-11-14T00:28:41,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/4ff23a46dc0641999464a5d063ad6795 is 50, key is test_row_0/B:col10/1731544121687/Put/seqid=0 2024-11-14T00:28:41,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742084_1260 (size=12301) 2024-11-14T00:28:41,826 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/4ff23a46dc0641999464a5d063ad6795 2024-11-14T00:28:41,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/0c7793094a654e14a5c0e392c8d198e9 is 50, key is test_row_0/C:col10/1731544121687/Put/seqid=0 2024-11-14T00:28:41,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544181844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:41,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544181846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:41,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544181855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:41,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:41,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742085_1261 (size=12301) 2024-11-14T00:28:41,875 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/0c7793094a654e14a5c0e392c8d198e9 2024-11-14T00:28:41,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/741203e72717487c98eb64afb2530ba5 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/741203e72717487c98eb64afb2530ba5 2024-11-14T00:28:41,890 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/741203e72717487c98eb64afb2530ba5, entries=150, sequenceid=265, filesize=12.0 K 2024-11-14T00:28:41,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/4ff23a46dc0641999464a5d063ad6795 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/4ff23a46dc0641999464a5d063ad6795 2024-11-14T00:28:41,956 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/4ff23a46dc0641999464a5d063ad6795, entries=150, sequenceid=265, filesize=12.0 K 2024-11-14T00:28:41,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544181955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:41,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/0c7793094a654e14a5c0e392c8d198e9 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/0c7793094a654e14a5c0e392c8d198e9 2024-11-14T00:28:41,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544181966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:41,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:41,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544181966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:41,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:41,998 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/0c7793094a654e14a5c0e392c8d198e9, entries=150, sequenceid=265, filesize=12.0 K 2024-11-14T00:28:42,000 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 328b146c51a69a897800000a78ffe3a7 in 307ms, sequenceid=265, compaction requested=true 2024-11-14T00:28:42,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:42,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:42,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-14T00:28:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-14T00:28:42,007 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-14T00:28:42,007 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3950 sec 2024-11-14T00:28:42,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.4090 sec 2024-11-14T00:28:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:42,175 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:42,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-14T00:28:42,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/a2c6d45ae3304789bcfcdd373a5a48de is 50, key is test_row_0/A:col10/1731544121773/Put/seqid=0 2024-11-14T00:28:42,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544182189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:42,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:42,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544182194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:42,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544182205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:42,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742086_1262 (size=12301) 2024-11-14T00:28:42,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:42,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544182308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:42,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:42,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:42,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544182316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:42,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:42,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:42,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544182316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:42,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544182525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:42,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:42,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544182525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:42,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:42,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544182545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:42,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/a2c6d45ae3304789bcfcdd373a5a48de 2024-11-14T00:28:42,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/04d83d35af8e4de381b9f8b33618ab36 is 50, key is test_row_0/B:col10/1731544121773/Put/seqid=0 2024-11-14T00:28:42,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742087_1263 (size=12301) 2024-11-14T00:28:42,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51624 deadline: 1731544182709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:42,713 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_0' in 328b146c51a69a897800000a78ffe3a7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8271 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:28:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:42,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544182836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544182836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:42,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544182856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:43,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/04d83d35af8e4de381b9f8b33618ab36 2024-11-14T00:28:43,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/a346a7f60d4442aeb6e6d91791bca7b8 is 50, key is test_row_0/C:col10/1731544121773/Put/seqid=0 2024-11-14T00:28:43,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742088_1264 (size=12301) 2024-11-14T00:28:43,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:43,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544183345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:43,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544183348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:43,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:43,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:43,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544183375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:43,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:43,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/a346a7f60d4442aeb6e6d91791bca7b8 2024-11-14T00:28:43,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/a2c6d45ae3304789bcfcdd373a5a48de as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a2c6d45ae3304789bcfcdd373a5a48de 2024-11-14T00:28:43,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a2c6d45ae3304789bcfcdd373a5a48de, entries=150, sequenceid=292, filesize=12.0 K 2024-11-14T00:28:43,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/04d83d35af8e4de381b9f8b33618ab36 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/04d83d35af8e4de381b9f8b33618ab36 2024-11-14T00:28:43,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/04d83d35af8e4de381b9f8b33618ab36, entries=150, sequenceid=292, filesize=12.0 K 2024-11-14T00:28:43,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/a346a7f60d4442aeb6e6d91791bca7b8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/a346a7f60d4442aeb6e6d91791bca7b8 2024-11-14T00:28:43,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/a346a7f60d4442aeb6e6d91791bca7b8, entries=150, sequenceid=292, filesize=12.0 K 2024-11-14T00:28:43,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 328b146c51a69a897800000a78ffe3a7 in 1447ms, sequenceid=292, compaction requested=true 2024-11-14T00:28:43,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:43,622 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:28:43,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:43,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:43,622 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:28:43,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:43,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:43,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:43,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:43,626 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:28:43,626 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/A is initiating minor compaction (all files) 2024-11-14T00:28:43,627 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/A in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:43,628 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/92cf30576118492f973ff11e2d0af48e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/f93335985b1c43c08241fe803f23543a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/741203e72717487c98eb64afb2530ba5, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a2c6d45ae3304789bcfcdd373a5a48de] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=48.3 K 2024-11-14T00:28:43,628 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:28:43,628 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/B is initiating minor compaction (all files) 2024-11-14T00:28:43,629 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/B in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:43,629 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d6ede8630e9b49a5a410279329894dfd, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d8128998796541dabde32176ddc20508, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/4ff23a46dc0641999464a5d063ad6795, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/04d83d35af8e4de381b9f8b33618ab36] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=48.3 K 2024-11-14T00:28:43,629 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 92cf30576118492f973ff11e2d0af48e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731544119675 2024-11-14T00:28:43,629 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6ede8630e9b49a5a410279329894dfd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731544119675 2024-11-14T00:28:43,630 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting f93335985b1c43c08241fe803f23543a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731544119831 2024-11-14T00:28:43,630 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8128998796541dabde32176ddc20508, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731544119831 2024-11-14T00:28:43,630 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 741203e72717487c98eb64afb2530ba5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731544120517 2024-11-14T00:28:43,630 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ff23a46dc0641999464a5d063ad6795, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731544120517 2024-11-14T00:28:43,631 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting a2c6d45ae3304789bcfcdd373a5a48de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731544121773 2024-11-14T00:28:43,632 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04d83d35af8e4de381b9f8b33618ab36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731544121773 2024-11-14T00:28:43,652 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#B#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:43,653 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/f7ab36cb533a45ccbf7fe2118fa7d6f8 is 50, key is test_row_0/B:col10/1731544121773/Put/seqid=0 2024-11-14T00:28:43,663 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#A#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:43,663 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/fd0aebc8c254403ab7868766195ee490 is 50, key is test_row_0/A:col10/1731544121773/Put/seqid=0 2024-11-14T00:28:43,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742090_1266 (size=12983) 2024-11-14T00:28:43,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742089_1265 (size=12983) 2024-11-14T00:28:43,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-14T00:28:43,792 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:43,793 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-14T00:28:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T00:28:43,800 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:43,801 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:43,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T00:28:43,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T00:28:43,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:43,957 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T00:28:43,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/be5b760ad863457cbd7243d4a5bb1bc9 is 50, key is test_row_0/A:col10/1731544122177/Put/seqid=0 2024-11-14T00:28:43,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742091_1267 (size=9857) 2024-11-14T00:28:43,983 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/be5b760ad863457cbd7243d4a5bb1bc9 2024-11-14T00:28:44,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/7ec80c2aba414e34ade37659cd3f98fb is 50, key is test_row_0/B:col10/1731544122177/Put/seqid=0 2024-11-14T00:28:44,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742092_1268 (size=9857) 2024-11-14T00:28:44,018 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/7ec80c2aba414e34ade37659cd3f98fb 2024-11-14T00:28:44,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/04af2697e3974f44a947eefa433eb78e is 50, key is test_row_0/C:col10/1731544122177/Put/seqid=0 2024-11-14T00:28:44,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742093_1269 (size=9857) 2024-11-14T00:28:44,107 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/f7ab36cb533a45ccbf7fe2118fa7d6f8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f7ab36cb533a45ccbf7fe2118fa7d6f8 2024-11-14T00:28:44,109 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/fd0aebc8c254403ab7868766195ee490 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/fd0aebc8c254403ab7868766195ee490 2024-11-14T00:28:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T00:28:44,117 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/A of 328b146c51a69a897800000a78ffe3a7 into fd0aebc8c254403ab7868766195ee490(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:44,117 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:44,117 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/A, priority=12, startTime=1731544123622; duration=0sec 2024-11-14T00:28:44,117 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:44,117 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:A 2024-11-14T00:28:44,117 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:28:44,118 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:28:44,118 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/C is initiating minor compaction (all files) 2024-11-14T00:28:44,118 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/C in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:44,118 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/dabe352cb1bc41c4b4b5b5659aa47f09, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/1c1416405ab649c3a7c522eaa3ac18a7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/0c7793094a654e14a5c0e392c8d198e9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/a346a7f60d4442aeb6e6d91791bca7b8] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=48.3 K 2024-11-14T00:28:44,119 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting dabe352cb1bc41c4b4b5b5659aa47f09, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731544119675 2024-11-14T00:28:44,120 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c1416405ab649c3a7c522eaa3ac18a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731544119831 2024-11-14T00:28:44,120 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/B of 328b146c51a69a897800000a78ffe3a7 into f7ab36cb533a45ccbf7fe2118fa7d6f8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:44,120 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:44,120 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/B, priority=12, startTime=1731544123622; duration=0sec 2024-11-14T00:28:44,121 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:44,121 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:B 2024-11-14T00:28:44,126 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c7793094a654e14a5c0e392c8d198e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731544120517 2024-11-14T00:28:44,127 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting a346a7f60d4442aeb6e6d91791bca7b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731544121773 2024-11-14T00:28:44,140 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#C#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:44,142 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/3908c933bf254e4db4432fbf284aed82 is 50, key is test_row_0/C:col10/1731544121773/Put/seqid=0 2024-11-14T00:28:44,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742094_1270 (size=12983) 2024-11-14T00:28:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:44,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T00:28:44,445 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/04af2697e3974f44a947eefa433eb78e 2024-11-14T00:28:44,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544184453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:44,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/be5b760ad863457cbd7243d4a5bb1bc9 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/be5b760ad863457cbd7243d4a5bb1bc9 2024-11-14T00:28:44,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544184460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:44,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544184460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:44,470 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/be5b760ad863457cbd7243d4a5bb1bc9, entries=100, sequenceid=302, filesize=9.6 K 2024-11-14T00:28:44,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/7ec80c2aba414e34ade37659cd3f98fb as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/7ec80c2aba414e34ade37659cd3f98fb 2024-11-14T00:28:44,481 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/7ec80c2aba414e34ade37659cd3f98fb, entries=100, sequenceid=302, filesize=9.6 K 2024-11-14T00:28:44,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/04af2697e3974f44a947eefa433eb78e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/04af2697e3974f44a947eefa433eb78e 2024-11-14T00:28:44,493 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/04af2697e3974f44a947eefa433eb78e, entries=100, sequenceid=302, filesize=9.6 K 2024-11-14T00:28:44,495 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 328b146c51a69a897800000a78ffe3a7 in 538ms, sequenceid=302, compaction requested=false 2024-11-14T00:28:44,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:44,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:44,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-14T00:28:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-14T00:28:44,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-14T00:28:44,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 696 msec 2024-11-14T00:28:44,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 709 msec 2024-11-14T00:28:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:44,572 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:44,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-14T00:28:44,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/d9dfde48ed2545428ab27ab629c204ce is 50, key is test_row_0/A:col10/1731544124457/Put/seqid=0 2024-11-14T00:28:44,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544184580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:44,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544184583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:44,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544184583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:44,593 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/3908c933bf254e4db4432fbf284aed82 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/3908c933bf254e4db4432fbf284aed82 2024-11-14T00:28:44,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742095_1271 (size=12301) 2024-11-14T00:28:44,598 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/C of 328b146c51a69a897800000a78ffe3a7 into 3908c933bf254e4db4432fbf284aed82(size=12.7 K), total size for store is 22.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:44,598 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:44,598 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/C, priority=12, startTime=1731544123622; duration=0sec 2024-11-14T00:28:44,598 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:44,598 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:C 2024-11-14T00:28:44,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544184696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:44,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544184696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:44,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544184696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:44,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:44,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544184905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:44,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544184905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:44,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544184905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:44,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T00:28:44,936 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:44,937 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:44,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-14T00:28:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T00:28:44,950 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:44,951 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:44,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:44,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/d9dfde48ed2545428ab27ab629c204ce 2024-11-14T00:28:45,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/f1d5b22905394dc09578b3382c02439f is 50, key is test_row_0/B:col10/1731544124457/Put/seqid=0 2024-11-14T00:28:45,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742096_1272 (size=12301) 2024-11-14T00:28:45,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/f1d5b22905394dc09578b3382c02439f 2024-11-14T00:28:45,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/ee98b5a34d344f28b69698acae1a5a27 is 50, key is test_row_0/C:col10/1731544124457/Put/seqid=0 2024-11-14T00:28:45,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T00:28:45,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742097_1273 (size=12301) 2024-11-14T00:28:45,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/ee98b5a34d344f28b69698acae1a5a27 2024-11-14T00:28:45,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/d9dfde48ed2545428ab27ab629c204ce as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d9dfde48ed2545428ab27ab629c204ce 2024-11-14T00:28:45,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d9dfde48ed2545428ab27ab629c204ce, entries=150, sequenceid=332, filesize=12.0 K 2024-11-14T00:28:45,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/f1d5b22905394dc09578b3382c02439f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f1d5b22905394dc09578b3382c02439f 2024-11-14T00:28:45,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f1d5b22905394dc09578b3382c02439f, entries=150, sequenceid=332, filesize=12.0 K 2024-11-14T00:28:45,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/ee98b5a34d344f28b69698acae1a5a27 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/ee98b5a34d344f28b69698acae1a5a27 2024-11-14T00:28:45,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/ee98b5a34d344f28b69698acae1a5a27, entries=150, sequenceid=332, filesize=12.0 K 2024-11-14T00:28:45,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 328b146c51a69a897800000a78ffe3a7 in 519ms, sequenceid=332, compaction requested=true 2024-11-14T00:28:45,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:45,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:45,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:45,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:45,091 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:45,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:45,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:45,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:28:45,091 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:45,092 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:45,092 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/B is initiating minor compaction (all files) 2024-11-14T00:28:45,092 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/B in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:45,093 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f7ab36cb533a45ccbf7fe2118fa7d6f8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/7ec80c2aba414e34ade37659cd3f98fb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f1d5b22905394dc09578b3382c02439f] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=34.3 K 2024-11-14T00:28:45,093 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:45,093 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/A is initiating minor compaction (all files) 2024-11-14T00:28:45,093 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/A in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:45,093 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/fd0aebc8c254403ab7868766195ee490, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/be5b760ad863457cbd7243d4a5bb1bc9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d9dfde48ed2545428ab27ab629c204ce] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=34.3 K 2024-11-14T00:28:45,094 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd0aebc8c254403ab7868766195ee490, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731544121773 2024-11-14T00:28:45,094 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting f7ab36cb533a45ccbf7fe2118fa7d6f8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731544121773 2024-11-14T00:28:45,094 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting be5b760ad863457cbd7243d4a5bb1bc9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731544122177 2024-11-14T00:28:45,095 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ec80c2aba414e34ade37659cd3f98fb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731544122177 2024-11-14T00:28:45,095 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9dfde48ed2545428ab27ab629c204ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731544124457 2024-11-14T00:28:45,095 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting f1d5b22905394dc09578b3382c02439f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731544124457 2024-11-14T00:28:45,105 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#A#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:45,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T00:28:45,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:45,106 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#B#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:45,106 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/072aa3397a9c48c7b6ebbc7f158fc0d1 is 50, key is test_row_0/A:col10/1731544124457/Put/seqid=0 2024-11-14T00:28:45,106 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-14T00:28:45,106 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/5740190e37674a94afa95c53992a2b9d is 50, key is test_row_0/B:col10/1731544124457/Put/seqid=0 2024-11-14T00:28:45,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/f0ffe2f4e1324de59f8da8cea47160d7 is 50, key is test_row_0/A:col10/1731544124582/Put/seqid=0 2024-11-14T00:28:45,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742098_1274 (size=13085) 2024-11-14T00:28:45,137 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/5740190e37674a94afa95c53992a2b9d as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/5740190e37674a94afa95c53992a2b9d 2024-11-14T00:28:45,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742100_1276 (size=9857) 2024-11-14T00:28:45,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742099_1275 (size=13085) 2024-11-14T00:28:45,139 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/f0ffe2f4e1324de59f8da8cea47160d7 2024-11-14T00:28:45,145 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/B of 328b146c51a69a897800000a78ffe3a7 into 5740190e37674a94afa95c53992a2b9d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:45,145 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:45,145 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/B, priority=13, startTime=1731544125091; duration=0sec 2024-11-14T00:28:45,146 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:45,146 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:B 2024-11-14T00:28:45,146 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:45,148 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/072aa3397a9c48c7b6ebbc7f158fc0d1 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/072aa3397a9c48c7b6ebbc7f158fc0d1 2024-11-14T00:28:45,148 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:45,148 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/C is initiating minor compaction (all files) 2024-11-14T00:28:45,148 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/C in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:45,149 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/3908c933bf254e4db4432fbf284aed82, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/04af2697e3974f44a947eefa433eb78e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/ee98b5a34d344f28b69698acae1a5a27] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=34.3 K 2024-11-14T00:28:45,150 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 3908c933bf254e4db4432fbf284aed82, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731544121773 2024-11-14T00:28:45,150 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 04af2697e3974f44a947eefa433eb78e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731544122177 2024-11-14T00:28:45,153 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting ee98b5a34d344f28b69698acae1a5a27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731544124457 2024-11-14T00:28:45,159 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/A of 328b146c51a69a897800000a78ffe3a7 into 072aa3397a9c48c7b6ebbc7f158fc0d1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:45,159 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:45,159 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/A, priority=13, startTime=1731544125091; duration=0sec 2024-11-14T00:28:45,160 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:45,160 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:A 2024-11-14T00:28:45,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/649a2967ef33408f91c62d38503eabd4 is 50, key is test_row_0/B:col10/1731544124582/Put/seqid=0 2024-11-14T00:28:45,167 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#C#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:45,167 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/ee23e377effb4a1bbe8ae90ddfe2d322 is 50, key is test_row_0/C:col10/1731544124457/Put/seqid=0 2024-11-14T00:28:45,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742101_1277 (size=9857) 2024-11-14T00:28:45,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742102_1278 (size=13085) 2024-11-14T00:28:45,202 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/ee23e377effb4a1bbe8ae90ddfe2d322 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/ee23e377effb4a1bbe8ae90ddfe2d322 2024-11-14T00:28:45,211 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/C of 328b146c51a69a897800000a78ffe3a7 into ee23e377effb4a1bbe8ae90ddfe2d322(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:45,211 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:45,211 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/C, priority=13, startTime=1731544125091; duration=0sec 2024-11-14T00:28:45,211 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:45,211 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:C 2024-11-14T00:28:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:45,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:45,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:45,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544185258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:45,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:45,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:45,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544185259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:45,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:45,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544185259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:45,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T00:28:45,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:45,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544185369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:45,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:45,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:45,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544185375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:45,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:45,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544185377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T00:28:45,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:45,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544185575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:45,584 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/649a2967ef33408f91c62d38503eabd4 2024-11-14T00:28:45,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544185585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:45,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:45,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544185585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:45,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:45,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/d0d7e320f5a34ae8adb582637b035bcb is 50, key is test_row_0/C:col10/1731544124582/Put/seqid=0 2024-11-14T00:28:45,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742103_1279 (size=9857) 2024-11-14T00:28:45,596 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/d0d7e320f5a34ae8adb582637b035bcb 2024-11-14T00:28:45,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/f0ffe2f4e1324de59f8da8cea47160d7 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/f0ffe2f4e1324de59f8da8cea47160d7 2024-11-14T00:28:45,609 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/f0ffe2f4e1324de59f8da8cea47160d7, entries=100, sequenceid=341, filesize=9.6 K 2024-11-14T00:28:45,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/649a2967ef33408f91c62d38503eabd4 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/649a2967ef33408f91c62d38503eabd4 2024-11-14T00:28:45,617 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/649a2967ef33408f91c62d38503eabd4, entries=100, sequenceid=341, filesize=9.6 K 2024-11-14T00:28:45,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/d0d7e320f5a34ae8adb582637b035bcb as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/d0d7e320f5a34ae8adb582637b035bcb 2024-11-14T00:28:45,640 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/d0d7e320f5a34ae8adb582637b035bcb, entries=100, sequenceid=341, filesize=9.6 K 2024-11-14T00:28:45,642 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for 328b146c51a69a897800000a78ffe3a7 in 536ms, sequenceid=341, compaction requested=false 2024-11-14T00:28:45,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:45,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:45,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-14T00:28:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-14T00:28:45,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-14T00:28:45,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 693 msec 2024-11-14T00:28:45,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 710 msec 2024-11-14T00:28:45,887 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:45,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-14T00:28:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:45,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/d21253472df04d128992eeb46756d8d0 is 50, key is test_row_0/A:col10/1731544125242/Put/seqid=0 2024-11-14T00:28:45,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742104_1280 (size=12301) 2024-11-14T00:28:45,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544185940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:45,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544185940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544185940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:46,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544186055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:46,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544186055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:46,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:46,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544186055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:46,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T00:28:46,085 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:46,086 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-14T00:28:46,087 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:46,088 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:46,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:46,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T00:28:46,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T00:28:46,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T00:28:46,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:46,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:46,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:46,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:46,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:46,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:46,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544186265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:46,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544186265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:46,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544186265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:46,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/d21253472df04d128992eeb46756d8d0 2024-11-14T00:28:46,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/03ce259e6e1a4d138901f84e30ab32c2 is 50, key is test_row_0/B:col10/1731544125242/Put/seqid=0 2024-11-14T00:28:46,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742105_1281 (size=12301) 2024-11-14T00:28:46,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T00:28:46,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:46,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:46,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:46,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:46,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T00:28:46,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T00:28:46,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:46,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:46,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:46,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:46,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:46,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544186575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:46,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:46,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544186575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:46,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:46,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:46,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544186575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:46,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:46,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:46,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:46,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T00:28:46,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:46,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:46,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:46,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:46,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T00:28:46,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/03ce259e6e1a4d138901f84e30ab32c2 2024-11-14T00:28:46,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/60f0739e137b4969af5c16b36990f8ae is 50, key is test_row_0/C:col10/1731544125242/Put/seqid=0 2024-11-14T00:28:46,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742106_1282 (size=12301) 2024-11-14T00:28:46,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/60f0739e137b4969af5c16b36990f8ae 2024-11-14T00:28:46,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/d21253472df04d128992eeb46756d8d0 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d21253472df04d128992eeb46756d8d0 2024-11-14T00:28:46,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d21253472df04d128992eeb46756d8d0, entries=150, sequenceid=374, filesize=12.0 K 2024-11-14T00:28:46,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/03ce259e6e1a4d138901f84e30ab32c2 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/03ce259e6e1a4d138901f84e30ab32c2 2024-11-14T00:28:46,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/03ce259e6e1a4d138901f84e30ab32c2, entries=150, sequenceid=374, filesize=12.0 K 2024-11-14T00:28:46,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/60f0739e137b4969af5c16b36990f8ae as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/60f0739e137b4969af5c16b36990f8ae 2024-11-14T00:28:46,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/60f0739e137b4969af5c16b36990f8ae, entries=150, sequenceid=374, filesize=12.0 K 2024-11-14T00:28:46,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=33.54 KB/34350 for 328b146c51a69a897800000a78ffe3a7 in 936ms, sequenceid=374, compaction requested=true 2024-11-14T00:28:46,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:46,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:46,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:46,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:46,823 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:46,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:46,823 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:46,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:46,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:46,825 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:46,825 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/B is initiating minor compaction (all files) 2024-11-14T00:28:46,825 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/B in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:46,825 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/5740190e37674a94afa95c53992a2b9d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/649a2967ef33408f91c62d38503eabd4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/03ce259e6e1a4d138901f84e30ab32c2] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=34.4 K 2024-11-14T00:28:46,826 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:46,826 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/A is initiating minor compaction (all files) 2024-11-14T00:28:46,826 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/A in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:46,826 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/072aa3397a9c48c7b6ebbc7f158fc0d1, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/f0ffe2f4e1324de59f8da8cea47160d7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d21253472df04d128992eeb46756d8d0] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=34.4 K 2024-11-14T00:28:46,826 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5740190e37674a94afa95c53992a2b9d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731544124457 2024-11-14T00:28:46,827 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 072aa3397a9c48c7b6ebbc7f158fc0d1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731544124457 2024-11-14T00:28:46,827 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 649a2967ef33408f91c62d38503eabd4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731544124582 2024-11-14T00:28:46,827 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting f0ffe2f4e1324de59f8da8cea47160d7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731544124582 2024-11-14T00:28:46,828 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03ce259e6e1a4d138901f84e30ab32c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731544125242 2024-11-14T00:28:46,828 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting d21253472df04d128992eeb46756d8d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731544125242 2024-11-14T00:28:46,845 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:46,846 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/6599a1ab5248483f83f552050c035b7f is 50, key is test_row_0/A:col10/1731544125242/Put/seqid=0 2024-11-14T00:28:46,852 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#B#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:46,852 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/49aafafd6e1d495c8bab89d3295e7dba is 50, key is test_row_0/B:col10/1731544125242/Put/seqid=0 2024-11-14T00:28:46,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T00:28:46,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:46,863 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-14T00:28:46,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742107_1283 (size=13187) 2024-11-14T00:28:46,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/dae5451834b44554aabe97afe1f24b01 is 50, key is test_row_0/A:col10/1731544125917/Put/seqid=0 2024-11-14T00:28:46,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742108_1284 (size=13187) 2024-11-14T00:28:46,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742109_1285 (size=12301) 2024-11-14T00:28:47,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:47,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:47,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:47,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544187155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:47,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:47,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544187155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:47,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:47,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:47,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:47,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544187161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:47,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T00:28:47,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544187287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:47,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544187287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:47,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:47,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544187288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:47,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:47,305 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/49aafafd6e1d495c8bab89d3295e7dba as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/49aafafd6e1d495c8bab89d3295e7dba 2024-11-14T00:28:47,306 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/6599a1ab5248483f83f552050c035b7f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/6599a1ab5248483f83f552050c035b7f 2024-11-14T00:28:47,321 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/dae5451834b44554aabe97afe1f24b01 2024-11-14T00:28:47,323 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/B of 328b146c51a69a897800000a78ffe3a7 into 49aafafd6e1d495c8bab89d3295e7dba(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:47,323 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:47,323 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/B, priority=13, startTime=1731544126823; duration=0sec 2024-11-14T00:28:47,324 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:47,324 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:B 2024-11-14T00:28:47,324 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:47,325 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/A of 328b146c51a69a897800000a78ffe3a7 into 6599a1ab5248483f83f552050c035b7f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:47,325 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:47,325 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/A, priority=13, startTime=1731544126823; duration=0sec 2024-11-14T00:28:47,325 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:47,325 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:A 2024-11-14T00:28:47,333 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:47,333 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/C is initiating minor compaction (all files) 2024-11-14T00:28:47,333 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/C in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:47,333 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/ee23e377effb4a1bbe8ae90ddfe2d322, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/d0d7e320f5a34ae8adb582637b035bcb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/60f0739e137b4969af5c16b36990f8ae] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=34.4 K 2024-11-14T00:28:47,336 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee23e377effb4a1bbe8ae90ddfe2d322, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731544124457 2024-11-14T00:28:47,340 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0d7e320f5a34ae8adb582637b035bcb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731544124582 2024-11-14T00:28:47,344 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60f0739e137b4969af5c16b36990f8ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731544125242 2024-11-14T00:28:47,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/14a78546eeed4f8e801fce2b195a44d2 is 50, key is test_row_0/B:col10/1731544125917/Put/seqid=0 2024-11-14T00:28:47,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742110_1286 (size=12301) 2024-11-14T00:28:47,388 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/14a78546eeed4f8e801fce2b195a44d2 2024-11-14T00:28:47,391 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#C#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:47,391 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/e390c23d2850421b989f5ee052ca80cc is 50, key is test_row_0/C:col10/1731544125242/Put/seqid=0 2024-11-14T00:28:47,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/5056b6e109ff413ba1df2e590f3bf8bb is 50, key is test_row_0/C:col10/1731544125917/Put/seqid=0 2024-11-14T00:28:47,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742111_1287 (size=13187) 2024-11-14T00:28:47,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742112_1288 (size=12301) 2024-11-14T00:28:47,467 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/5056b6e109ff413ba1df2e590f3bf8bb 2024-11-14T00:28:47,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/dae5451834b44554aabe97afe1f24b01 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/dae5451834b44554aabe97afe1f24b01 2024-11-14T00:28:47,477 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/dae5451834b44554aabe97afe1f24b01, entries=150, sequenceid=382, filesize=12.0 K 2024-11-14T00:28:47,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/14a78546eeed4f8e801fce2b195a44d2 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/14a78546eeed4f8e801fce2b195a44d2 2024-11-14T00:28:47,492 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/14a78546eeed4f8e801fce2b195a44d2, entries=150, sequenceid=382, filesize=12.0 K 2024-11-14T00:28:47,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/5056b6e109ff413ba1df2e590f3bf8bb as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/5056b6e109ff413ba1df2e590f3bf8bb 2024-11-14T00:28:47,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544187495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:47,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544187495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:47,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544187495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:47,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:47,510 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/5056b6e109ff413ba1df2e590f3bf8bb, entries=150, sequenceid=382, filesize=12.0 K 2024-11-14T00:28:47,515 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 328b146c51a69a897800000a78ffe3a7 in 652ms, sequenceid=382, compaction requested=false 2024-11-14T00:28:47,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:47,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:47,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-14T00:28:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-14T00:28:47,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-14T00:28:47,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4290 sec 2024-11-14T00:28:47,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4360 sec 2024-11-14T00:28:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:47,809 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:47,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-14T00:28:47,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544187815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:47,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/16c9efb3980b423e9438f3a8fc6f4a1c is 50, key is test_row_0/A:col10/1731544127807/Put/seqid=0 2024-11-14T00:28:47,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544187817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:47,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544187817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:47,841 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/e390c23d2850421b989f5ee052ca80cc as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e390c23d2850421b989f5ee052ca80cc 2024-11-14T00:28:47,849 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/C of 328b146c51a69a897800000a78ffe3a7 into e390c23d2850421b989f5ee052ca80cc(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:47,849 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:47,849 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/C, priority=13, startTime=1731544126823; duration=0sec 2024-11-14T00:28:47,849 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:47,849 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:C 2024-11-14T00:28:47,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742113_1289 (size=14741) 2024-11-14T00:28:47,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544187925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:47,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:47,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544187925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:47,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544187925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:48,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544188136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:48,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544188136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:48,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544188136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T00:28:48,236 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:48,237 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-14T00:28:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T00:28:48,252 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:48,254 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:48,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:48,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/16c9efb3980b423e9438f3a8fc6f4a1c 2024-11-14T00:28:48,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/c90342ca322d4d04babc1860628ac216 is 50, key is test_row_0/B:col10/1731544127807/Put/seqid=0 2024-11-14T00:28:48,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742114_1290 (size=12301) 2024-11-14T00:28:48,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/c90342ca322d4d04babc1860628ac216 2024-11-14T00:28:48,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/c7f729d6e0834aec9905bef21453b27b is 50, key is test_row_0/C:col10/1731544127807/Put/seqid=0 2024-11-14T00:28:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T00:28:48,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742115_1291 (size=12301) 2024-11-14T00:28:48,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/c7f729d6e0834aec9905bef21453b27b 2024-11-14T00:28:48,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T00:28:48,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:48,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:48,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:48,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:48,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:48,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/16c9efb3980b423e9438f3a8fc6f4a1c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/16c9efb3980b423e9438f3a8fc6f4a1c 2024-11-14T00:28:48,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/16c9efb3980b423e9438f3a8fc6f4a1c, entries=200, sequenceid=415, filesize=14.4 K 2024-11-14T00:28:48,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/c90342ca322d4d04babc1860628ac216 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/c90342ca322d4d04babc1860628ac216 2024-11-14T00:28:48,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544188445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:48,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:48,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:48,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544188445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:48,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:48,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:48,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:48,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:48,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544188448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:48,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:48,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:48,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:48,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/c90342ca322d4d04babc1860628ac216, entries=150, sequenceid=415, filesize=12.0 K 2024-11-14T00:28:48,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/c7f729d6e0834aec9905bef21453b27b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c7f729d6e0834aec9905bef21453b27b 2024-11-14T00:28:48,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c7f729d6e0834aec9905bef21453b27b, entries=150, sequenceid=415, filesize=12.0 K 2024-11-14T00:28:48,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 328b146c51a69a897800000a78ffe3a7 in 667ms, sequenceid=415, compaction requested=true 2024-11-14T00:28:48,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:48,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:48,476 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:48,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:48,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:48,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:48,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:48,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:28:48,476 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:48,478 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:48,478 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/A is initiating minor compaction (all files) 2024-11-14T00:28:48,478 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/A in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:48,479 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/6599a1ab5248483f83f552050c035b7f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/dae5451834b44554aabe97afe1f24b01, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/16c9efb3980b423e9438f3a8fc6f4a1c] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=39.3 K 2024-11-14T00:28:48,479 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:48,479 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/B is initiating minor compaction (all files) 2024-11-14T00:28:48,479 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/B in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:48,479 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/49aafafd6e1d495c8bab89d3295e7dba, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/14a78546eeed4f8e801fce2b195a44d2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/c90342ca322d4d04babc1860628ac216] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=36.9 K 2024-11-14T00:28:48,479 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6599a1ab5248483f83f552050c035b7f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731544125242 2024-11-14T00:28:48,479 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting dae5451834b44554aabe97afe1f24b01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731544125892 2024-11-14T00:28:48,479 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 49aafafd6e1d495c8bab89d3295e7dba, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731544125242 2024-11-14T00:28:48,480 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16c9efb3980b423e9438f3a8fc6f4a1c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731544127151 2024-11-14T00:28:48,480 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 14a78546eeed4f8e801fce2b195a44d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731544125892 2024-11-14T00:28:48,480 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c90342ca322d4d04babc1860628ac216, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731544127807 2024-11-14T00:28:48,517 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:48,517 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/133e7bc3d42541e3b4585088119d3ee1 is 50, key is test_row_0/B:col10/1731544127807/Put/seqid=0 2024-11-14T00:28:48,518 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:48,518 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/12a1f0b6831a40f99393debbb8c012d4 is 50, key is test_row_0/A:col10/1731544127807/Put/seqid=0 2024-11-14T00:28:48,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742116_1292 (size=13289) 2024-11-14T00:28:48,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742117_1293 (size=13289) 2024-11-14T00:28:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T00:28:48,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T00:28:48,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:48,573 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-14T00:28:48,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/923b7361e0fd4e93b414ccd70b17daa7 is 50, key is test_row_2/A:col10/1731544127815/Put/seqid=0 2024-11-14T00:28:48,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742118_1294 (size=7415) 2024-11-14T00:28:48,593 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/923b7361e0fd4e93b414ccd70b17daa7 2024-11-14T00:28:48,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/c93230932a5942c0b1206550de361af6 is 50, key is test_row_2/B:col10/1731544127815/Put/seqid=0 2024-11-14T00:28:48,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742119_1295 (size=7415) 2024-11-14T00:28:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T00:28:48,953 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/133e7bc3d42541e3b4585088119d3ee1 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/133e7bc3d42541e3b4585088119d3ee1 2024-11-14T00:28:48,962 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/B of 328b146c51a69a897800000a78ffe3a7 into 133e7bc3d42541e3b4585088119d3ee1(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:48,963 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:48,963 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/B, priority=13, startTime=1731544128476; duration=0sec 2024-11-14T00:28:48,963 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:48,963 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:B 2024-11-14T00:28:48,963 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:48,965 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:48,965 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/C is initiating minor compaction (all files) 2024-11-14T00:28:48,965 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/C in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:48,966 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e390c23d2850421b989f5ee052ca80cc, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/5056b6e109ff413ba1df2e590f3bf8bb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c7f729d6e0834aec9905bef21453b27b] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=36.9 K 2024-11-14T00:28:48,966 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting e390c23d2850421b989f5ee052ca80cc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731544125242 2024-11-14T00:28:48,967 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 5056b6e109ff413ba1df2e590f3bf8bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731544125892 2024-11-14T00:28:48,968 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/12a1f0b6831a40f99393debbb8c012d4 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/12a1f0b6831a40f99393debbb8c012d4 2024-11-14T00:28:48,968 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c7f729d6e0834aec9905bef21453b27b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731544127807 2024-11-14T00:28:48,978 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/A of 328b146c51a69a897800000a78ffe3a7 into 12a1f0b6831a40f99393debbb8c012d4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:48,978 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:48,978 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/A, priority=13, startTime=1731544128476; duration=0sec 2024-11-14T00:28:48,979 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:48,979 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:A 2024-11-14T00:28:48,980 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#C#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:48,981 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/c5ce9da8029e49fa83e97b76661e5d79 is 50, key is test_row_0/C:col10/1731544127807/Put/seqid=0 2024-11-14T00:28:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:48,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:49,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742120_1296 (size=13289) 2024-11-14T00:28:49,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544189060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:49,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544189061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:49,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544189061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:49,101 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/c93230932a5942c0b1206550de361af6 2024-11-14T00:28:49,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/4d58504660dc4c1bab2a81ab037913cb is 50, key is test_row_2/C:col10/1731544127815/Put/seqid=0 2024-11-14T00:28:49,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742121_1297 (size=7415) 2024-11-14T00:28:49,156 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/4d58504660dc4c1bab2a81ab037913cb 2024-11-14T00:28:49,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/923b7361e0fd4e93b414ccd70b17daa7 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/923b7361e0fd4e93b414ccd70b17daa7 2024-11-14T00:28:49,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:49,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544189165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:49,179 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/923b7361e0fd4e93b414ccd70b17daa7, entries=50, sequenceid=421, filesize=7.2 K 2024-11-14T00:28:49,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:49,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544189175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:49,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:49,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:49,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544189175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:49,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:49,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/c93230932a5942c0b1206550de361af6 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/c93230932a5942c0b1206550de361af6 2024-11-14T00:28:49,196 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/c93230932a5942c0b1206550de361af6, entries=50, sequenceid=421, filesize=7.2 K 2024-11-14T00:28:49,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/4d58504660dc4c1bab2a81ab037913cb as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/4d58504660dc4c1bab2a81ab037913cb 2024-11-14T00:28:49,205 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/4d58504660dc4c1bab2a81ab037913cb, entries=50, sequenceid=421, filesize=7.2 K 2024-11-14T00:28:49,211 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=194.56 KB/199230 for 328b146c51a69a897800000a78ffe3a7 in 636ms, sequenceid=421, compaction requested=false 2024-11-14T00:28:49,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:49,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:49,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-14T00:28:49,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-14T00:28:49,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-14T00:28:49,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 962 msec 2024-11-14T00:28:49,244 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 999 msec 2024-11-14T00:28:49,380 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:49,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=201.27 KB heapSize=528.09 KB 2024-11-14T00:28:49,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T00:28:49,386 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:49,386 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-14T00:28:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T00:28:49,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:49,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544189383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:49,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:49,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544189386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:49,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544189386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:49,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/bf63443d689a4a1584cecd664c2c535e is 50, key is test_row_0/A:col10/1731544129053/Put/seqid=0 2024-11-14T00:28:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:49,393 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:49,394 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:49,395 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:49,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742122_1298 (size=14741) 2024-11-14T00:28:49,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/bf63443d689a4a1584cecd664c2c535e 2024-11-14T00:28:49,469 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/c5ce9da8029e49fa83e97b76661e5d79 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c5ce9da8029e49fa83e97b76661e5d79 2024-11-14T00:28:49,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/4aa88f0c171540678212ca49b11a4714 is 50, key is test_row_0/B:col10/1731544129053/Put/seqid=0 2024-11-14T00:28:49,485 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/C of 328b146c51a69a897800000a78ffe3a7 into c5ce9da8029e49fa83e97b76661e5d79(size=13.0 K), total size for store is 20.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:49,485 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:49,485 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/C, priority=13, startTime=1731544128476; duration=0sec 2024-11-14T00:28:49,485 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:49,485 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:C 2024-11-14T00:28:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T00:28:49,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544189500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:49,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:49,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742123_1299 (size=12301) 2024-11-14T00:28:49,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/4aa88f0c171540678212ca49b11a4714 2024-11-14T00:28:49,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T00:28:49,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:49,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:49,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:49,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:49,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:49,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/6a40bd0e3a244263ab9983838c855dd2 is 50, key is test_row_0/C:col10/1731544129053/Put/seqid=0 2024-11-14T00:28:49,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742124_1300 (size=12301) 2024-11-14T00:28:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T00:28:49,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51638 deadline: 1731544189705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:49,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51614 deadline: 1731544189706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:49,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T00:28:49,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:49,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:49,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:49,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:49,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:49,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544189716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:49,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T00:28:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:49,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:49,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:49,940 INFO [Thread-987 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-987. 2024-11-14T00:28:49,940 DEBUG [Thread-987 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:49,940 DEBUG [Thread-987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:49,941 DEBUG [Thread-987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:49,941 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:49,941 INFO [Thread-989 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-989. 2024-11-14T00:28:49,941 DEBUG [Thread-989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:49,941 INFO [Thread-993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-993. 2024-11-14T00:28:49,941 DEBUG [Thread-989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:49,941 DEBUG [Thread-993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:49,941 DEBUG [Thread-993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:49,941 DEBUG [Thread-989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:49,941 DEBUG [Thread-993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:49,942 INFO [Thread-991 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-991. 2024-11-14T00:28:49,942 DEBUG [Thread-991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:49,942 DEBUG [Thread-991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:49,943 DEBUG [Thread-991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:49,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:49,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:49,943 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:49,946 INFO [Thread-985 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-985. 2024-11-14T00:28:49,946 DEBUG [Thread-985 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:49,947 DEBUG [Thread-985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:49,947 DEBUG [Thread-985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:49,947 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:50,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/6a40bd0e3a244263ab9983838c855dd2 2024-11-14T00:28:50,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/bf63443d689a4a1584cecd664c2c535e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/bf63443d689a4a1584cecd664c2c535e 2024-11-14T00:28:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T00:28:50,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/bf63443d689a4a1584cecd664c2c535e, entries=200, sequenceid=456, filesize=14.4 K 2024-11-14T00:28:50,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T00:28:50,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:50,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. as already flushing 2024-11-14T00:28:50,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:50,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/4aa88f0c171540678212ca49b11a4714 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/4aa88f0c171540678212ca49b11a4714 2024-11-14T00:28:50,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:50,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:50,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51620 deadline: 1731544190025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:50,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:50,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:50,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:28:50,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/4aa88f0c171540678212ca49b11a4714, entries=150, sequenceid=456, filesize=12.0 K 2024-11-14T00:28:50,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/6a40bd0e3a244263ab9983838c855dd2 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/6a40bd0e3a244263ab9983838c855dd2 2024-11-14T00:28:50,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/6a40bd0e3a244263ab9983838c855dd2, entries=150, sequenceid=456, filesize=12.0 K 2024-11-14T00:28:50,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for 328b146c51a69a897800000a78ffe3a7 in 656ms, sequenceid=456, compaction requested=true 2024-11-14T00:28:50,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:50,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:28:50,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:50,036 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:50,036 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:50,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:28:50,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:50,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 328b146c51a69a897800000a78ffe3a7:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:28:50,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:50,037 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35445 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:50,037 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:50,037 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/A is initiating minor compaction (all files) 2024-11-14T00:28:50,037 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/B is initiating minor compaction (all files) 2024-11-14T00:28:50,038 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/A in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:50,038 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/B in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:50,038 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/133e7bc3d42541e3b4585088119d3ee1, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/c93230932a5942c0b1206550de361af6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/4aa88f0c171540678212ca49b11a4714] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=32.2 K 2024-11-14T00:28:50,038 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/12a1f0b6831a40f99393debbb8c012d4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/923b7361e0fd4e93b414ccd70b17daa7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/bf63443d689a4a1584cecd664c2c535e] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=34.6 K 2024-11-14T00:28:50,038 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 133e7bc3d42541e3b4585088119d3ee1, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731544127807 2024-11-14T00:28:50,038 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12a1f0b6831a40f99393debbb8c012d4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731544127807 2024-11-14T00:28:50,038 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 923b7361e0fd4e93b414ccd70b17daa7, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731544127815 2024-11-14T00:28:50,038 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c93230932a5942c0b1206550de361af6, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731544127815 2024-11-14T00:28:50,039 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf63443d689a4a1584cecd664c2c535e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731544129047 2024-11-14T00:28:50,039 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aa88f0c171540678212ca49b11a4714, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731544129047 2024-11-14T00:28:50,054 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#A#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:50,054 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:50,055 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/88fae82f91c74eadb42f4a8fc7ad223a is 50, key is test_row_0/B:col10/1731544129053/Put/seqid=0 2024-11-14T00:28:50,055 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/28000251782245989710c5b05e303798 is 50, key is test_row_0/A:col10/1731544129053/Put/seqid=0 2024-11-14T00:28:50,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742126_1302 (size=13391) 2024-11-14T00:28:50,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742125_1301 (size=13391) 2024-11-14T00:28:50,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T00:28:50,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:50,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:50,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:50,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-14T00:28:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-14T00:28:50,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-14T00:28:50,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 789 msec 2024-11-14T00:28:50,204 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 814 msec 2024-11-14T00:28:50,218 INFO [Thread-976 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-976. 2024-11-14T00:28:50,218 DEBUG [Thread-976 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:50,218 DEBUG [Thread-976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:50,218 DEBUG [Thread-976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:50,220 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:50,220 INFO [Thread-974 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-974. 2024-11-14T00:28:50,220 DEBUG [Thread-974 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:50,220 DEBUG [Thread-974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:50,220 DEBUG [Thread-974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:50,220 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:50,465 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/88fae82f91c74eadb42f4a8fc7ad223a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/88fae82f91c74eadb42f4a8fc7ad223a 2024-11-14T00:28:50,470 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/28000251782245989710c5b05e303798 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/28000251782245989710c5b05e303798 2024-11-14T00:28:50,471 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/B of 328b146c51a69a897800000a78ffe3a7 into 88fae82f91c74eadb42f4a8fc7ad223a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:50,471 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:50,471 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/B, priority=13, startTime=1731544130036; duration=0sec 2024-11-14T00:28:50,471 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:28:50,471 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:B 2024-11-14T00:28:50,471 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:28:50,473 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:28:50,473 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 328b146c51a69a897800000a78ffe3a7/C is initiating minor compaction (all files) 2024-11-14T00:28:50,473 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 328b146c51a69a897800000a78ffe3a7/C in TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:50,473 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c5ce9da8029e49fa83e97b76661e5d79, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/4d58504660dc4c1bab2a81ab037913cb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/6a40bd0e3a244263ab9983838c855dd2] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp, totalSize=32.2 K 2024-11-14T00:28:50,474 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c5ce9da8029e49fa83e97b76661e5d79, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731544127807 2024-11-14T00:28:50,474 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d58504660dc4c1bab2a81ab037913cb, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731544127815 2024-11-14T00:28:50,474 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a40bd0e3a244263ab9983838c855dd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731544129047 2024-11-14T00:28:50,483 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/A of 328b146c51a69a897800000a78ffe3a7 into 28000251782245989710c5b05e303798(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:50,483 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:50,483 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/A, priority=13, startTime=1731544130036; duration=0sec 2024-11-14T00:28:50,483 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:50,483 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:A 2024-11-14T00:28:50,488 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 328b146c51a69a897800000a78ffe3a7#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:28:50,488 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/1f56133572664c4b940925fbdc1d3e25 is 50, key is test_row_0/C:col10/1731544129053/Put/seqid=0 2024-11-14T00:28:50,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742127_1303 (size=13391) 2024-11-14T00:28:50,508 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/1f56133572664c4b940925fbdc1d3e25 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/1f56133572664c4b940925fbdc1d3e25 2024-11-14T00:28:50,516 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 328b146c51a69a897800000a78ffe3a7/C of 328b146c51a69a897800000a78ffe3a7 into 1f56133572664c4b940925fbdc1d3e25(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:28:50,516 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 328b146c51a69a897800000a78ffe3a7: 2024-11-14T00:28:50,516 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., storeName=328b146c51a69a897800000a78ffe3a7/C, priority=13, startTime=1731544130037; duration=0sec 2024-11-14T00:28:50,516 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:28:50,516 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 328b146c51a69a897800000a78ffe3a7:C 2024-11-14T00:28:50,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T00:28:50,525 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:50,536 INFO [Thread-980 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-980. 2024-11-14T00:28:50,536 DEBUG [Thread-980 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:50,536 DEBUG [Thread-980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:50,536 DEBUG [Thread-980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:50,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:51,637 INFO [Thread-978 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-978. 2024-11-14T00:28:51,637 DEBUG [Thread-978 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:51,637 DEBUG [Thread-978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:51,637 DEBUG [Thread-978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:51,640 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:52,727 INFO [Thread-982 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-982. 2024-11-14T00:28:52,727 DEBUG [Thread-982 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:28:52,727 DEBUG [Thread-982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:52,727 DEBUG [Thread-982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:52,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:52,727 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T00:28:52,727 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-14T00:28:52,727 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-14T00:28:52,727 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-14T00:28:52,727 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-14T00:28:52,727 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-14T00:28:52,727 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T00:28:52,727 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2813 2024-11-14T00:28:52,727 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2807 2024-11-14T00:28:52,727 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2688 2024-11-14T00:28:52,727 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2820 2024-11-14T00:28:52,727 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2787 2024-11-14T00:28:52,727 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T00:28:52,727 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T00:28:52,727 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T00:28:52,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:52,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:52,728 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:28:52,729 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-14T00:28:52,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:52,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T00:28:52,734 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544132734"}]},"ts":"1731544132734"} 2024-11-14T00:28:52,736 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T00:28:52,736 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T00:28:52,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T00:28:52,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=328b146c51a69a897800000a78ffe3a7, UNASSIGN}] 2024-11-14T00:28:52,741 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=328b146c51a69a897800000a78ffe3a7, UNASSIGN 2024-11-14T00:28:52,742 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=328b146c51a69a897800000a78ffe3a7, regionState=CLOSING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:52,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=328b146c51a69a897800000a78ffe3a7, UNASSIGN because future has completed 2024-11-14T00:28:52,744 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T00:28:52,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:28:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T00:28:52,897 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:52,897 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T00:28:52,898 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing 328b146c51a69a897800000a78ffe3a7, disabling compactions & flushes 2024-11-14T00:28:52,898 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:52,898 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:52,898 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. after waiting 0 ms 2024-11-14T00:28:52,898 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:52,898 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(2902): Flushing 328b146c51a69a897800000a78ffe3a7 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-14T00:28:52,909 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/579f4ed2c89340a7a6ce85a31e14b890 is 50, key is test_row_0/A:col10/1731544132725/Put/seqid=0 2024-11-14T00:28:52,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742128_1304 (size=9857) 2024-11-14T00:28:52,953 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/579f4ed2c89340a7a6ce85a31e14b890 2024-11-14T00:28:52,978 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/387ff9e3f4534fc7885e441ba89d86b0 is 50, key is test_row_0/B:col10/1731544132725/Put/seqid=0 2024-11-14T00:28:53,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742129_1305 (size=9857) 2024-11-14T00:28:53,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T00:28:53,207 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T00:28:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T00:28:53,415 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/387ff9e3f4534fc7885e441ba89d86b0 2024-11-14T00:28:53,430 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/1d1e7d89029e48818319c12f825fa62c is 50, key is test_row_0/C:col10/1731544132725/Put/seqid=0 2024-11-14T00:28:53,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742130_1306 (size=9857) 2024-11-14T00:28:53,839 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/1d1e7d89029e48818319c12f825fa62c 2024-11-14T00:28:53,849 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/A/579f4ed2c89340a7a6ce85a31e14b890 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/579f4ed2c89340a7a6ce85a31e14b890 2024-11-14T00:28:53,854 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/579f4ed2c89340a7a6ce85a31e14b890, entries=100, sequenceid=469, filesize=9.6 K 2024-11-14T00:28:53,857 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/B/387ff9e3f4534fc7885e441ba89d86b0 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/387ff9e3f4534fc7885e441ba89d86b0 2024-11-14T00:28:53,862 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/387ff9e3f4534fc7885e441ba89d86b0, entries=100, sequenceid=469, filesize=9.6 K 2024-11-14T00:28:53,864 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/.tmp/C/1d1e7d89029e48818319c12f825fa62c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/1d1e7d89029e48818319c12f825fa62c 2024-11-14T00:28:53,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T00:28:53,869 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/1d1e7d89029e48818319c12f825fa62c, entries=100, sequenceid=469, filesize=9.6 K 2024-11-14T00:28:53,870 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 328b146c51a69a897800000a78ffe3a7 in 972ms, sequenceid=469, compaction requested=false 2024-11-14T00:28:53,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/00e2c348637e4272856f71dc8bc3cf2d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/7e9f9ab83d3c41aeb3320c9cd0ae9bb9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/111994165bf64b7b84bc98007f83ea41, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/23de917087e3429e87ce241948844434, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d215ea7ca0d54556a8e0fcb96ab4e6f4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/c502a522fea4455cb9c59cf4010e0180, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/ddd24d6a42ca446baa1976fdf8842732, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/42c5f79443e94362901be2a06cefae4f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/ebddf344a8f347be9d346029c991c05b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/c896e068548d4b7a85c5fabcec94c08f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a8685dd4836a425887225de5269b0234, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a9d32fb5312448968e63e9a9590e9e45, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/bb042bf553cd483abb5cd0c52d35c5e3, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/e0a7406a80f34fde8f3435985a565595, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/737e4132d8594304b45f605a7be04d5e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/92cf30576118492f973ff11e2d0af48e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/46422312941040648cbf369a2ee24bee, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/f93335985b1c43c08241fe803f23543a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/741203e72717487c98eb64afb2530ba5, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/fd0aebc8c254403ab7868766195ee490, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a2c6d45ae3304789bcfcdd373a5a48de, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/be5b760ad863457cbd7243d4a5bb1bc9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/072aa3397a9c48c7b6ebbc7f158fc0d1, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d9dfde48ed2545428ab27ab629c204ce, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/f0ffe2f4e1324de59f8da8cea47160d7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/6599a1ab5248483f83f552050c035b7f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d21253472df04d128992eeb46756d8d0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/dae5451834b44554aabe97afe1f24b01, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/16c9efb3980b423e9438f3a8fc6f4a1c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/12a1f0b6831a40f99393debbb8c012d4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/923b7361e0fd4e93b414ccd70b17daa7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/bf63443d689a4a1584cecd664c2c535e] to archive 2024-11-14T00:28:53,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:28:53,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/00e2c348637e4272856f71dc8bc3cf2d to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/00e2c348637e4272856f71dc8bc3cf2d 2024-11-14T00:28:53,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/7e9f9ab83d3c41aeb3320c9cd0ae9bb9 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/7e9f9ab83d3c41aeb3320c9cd0ae9bb9 2024-11-14T00:28:53,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/111994165bf64b7b84bc98007f83ea41 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/111994165bf64b7b84bc98007f83ea41 2024-11-14T00:28:53,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/23de917087e3429e87ce241948844434 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/23de917087e3429e87ce241948844434 2024-11-14T00:28:53,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d215ea7ca0d54556a8e0fcb96ab4e6f4 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d215ea7ca0d54556a8e0fcb96ab4e6f4 2024-11-14T00:28:53,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/c502a522fea4455cb9c59cf4010e0180 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/c502a522fea4455cb9c59cf4010e0180 2024-11-14T00:28:53,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/ddd24d6a42ca446baa1976fdf8842732 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/ddd24d6a42ca446baa1976fdf8842732 2024-11-14T00:28:53,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/42c5f79443e94362901be2a06cefae4f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/42c5f79443e94362901be2a06cefae4f 2024-11-14T00:28:53,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/ebddf344a8f347be9d346029c991c05b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/ebddf344a8f347be9d346029c991c05b 2024-11-14T00:28:53,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/c896e068548d4b7a85c5fabcec94c08f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/c896e068548d4b7a85c5fabcec94c08f 2024-11-14T00:28:53,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a8685dd4836a425887225de5269b0234 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a8685dd4836a425887225de5269b0234 2024-11-14T00:28:53,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a9d32fb5312448968e63e9a9590e9e45 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a9d32fb5312448968e63e9a9590e9e45 2024-11-14T00:28:53,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/bb042bf553cd483abb5cd0c52d35c5e3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/bb042bf553cd483abb5cd0c52d35c5e3 2024-11-14T00:28:53,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/e0a7406a80f34fde8f3435985a565595 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/e0a7406a80f34fde8f3435985a565595 2024-11-14T00:28:53,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/737e4132d8594304b45f605a7be04d5e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/737e4132d8594304b45f605a7be04d5e 2024-11-14T00:28:53,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/92cf30576118492f973ff11e2d0af48e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/92cf30576118492f973ff11e2d0af48e 2024-11-14T00:28:53,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/46422312941040648cbf369a2ee24bee to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/46422312941040648cbf369a2ee24bee 2024-11-14T00:28:53,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/f93335985b1c43c08241fe803f23543a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/f93335985b1c43c08241fe803f23543a 2024-11-14T00:28:53,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/741203e72717487c98eb64afb2530ba5 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/741203e72717487c98eb64afb2530ba5 2024-11-14T00:28:53,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/fd0aebc8c254403ab7868766195ee490 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/fd0aebc8c254403ab7868766195ee490 2024-11-14T00:28:53,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a2c6d45ae3304789bcfcdd373a5a48de to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/a2c6d45ae3304789bcfcdd373a5a48de 2024-11-14T00:28:53,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/be5b760ad863457cbd7243d4a5bb1bc9 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/be5b760ad863457cbd7243d4a5bb1bc9 2024-11-14T00:28:53,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/072aa3397a9c48c7b6ebbc7f158fc0d1 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/072aa3397a9c48c7b6ebbc7f158fc0d1 2024-11-14T00:28:53,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d9dfde48ed2545428ab27ab629c204ce to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d9dfde48ed2545428ab27ab629c204ce 2024-11-14T00:28:53,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/f0ffe2f4e1324de59f8da8cea47160d7 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/f0ffe2f4e1324de59f8da8cea47160d7 2024-11-14T00:28:53,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/6599a1ab5248483f83f552050c035b7f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/6599a1ab5248483f83f552050c035b7f 2024-11-14T00:28:53,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d21253472df04d128992eeb46756d8d0 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/d21253472df04d128992eeb46756d8d0 2024-11-14T00:28:53,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/dae5451834b44554aabe97afe1f24b01 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/dae5451834b44554aabe97afe1f24b01 2024-11-14T00:28:54,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/16c9efb3980b423e9438f3a8fc6f4a1c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/16c9efb3980b423e9438f3a8fc6f4a1c 2024-11-14T00:28:54,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/12a1f0b6831a40f99393debbb8c012d4 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/12a1f0b6831a40f99393debbb8c012d4 2024-11-14T00:28:54,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/923b7361e0fd4e93b414ccd70b17daa7 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/923b7361e0fd4e93b414ccd70b17daa7 2024-11-14T00:28:54,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/bf63443d689a4a1584cecd664c2c535e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/bf63443d689a4a1584cecd664c2c535e 2024-11-14T00:28:54,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/9a38c8b8998349f7b3f54fac21190878, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/1606cd57cfc24183a3e37c302002947b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/52cc026e0c3040bbaec7aaae4797dd01, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f7f7cdf3ba2c4f54a2ff0112b7031c41, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/ce3153ec72cc42eaadc5a30fda5658c8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/49f23a428adf4cc08728bfae57e3da27, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/2eae2ae581ff4d13bbf4605b6ececa6c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/8626967a3af64623832bff64143199da, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/01cfb3e56cce4a00b67128fa6eb3a3f9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/01d9cfaba4b74466bafda523025a66bc, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/3f47874e343e49639c7fc4898831a2a0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/2ad93d65b94940e7be8e4c60707e6bb1, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/ac4d756817ea420c8401a4bc78493611, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/6fb69cd9e5ab48ddae083c36d51ad4ff, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/aa0e152083684eb184593a4c53fed5b9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d6ede8630e9b49a5a410279329894dfd, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d4c49b76d7c34c678c70add0098ec995, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d8128998796541dabde32176ddc20508, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/4ff23a46dc0641999464a5d063ad6795, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f7ab36cb533a45ccbf7fe2118fa7d6f8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/04d83d35af8e4de381b9f8b33618ab36, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/7ec80c2aba414e34ade37659cd3f98fb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/5740190e37674a94afa95c53992a2b9d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f1d5b22905394dc09578b3382c02439f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/649a2967ef33408f91c62d38503eabd4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/49aafafd6e1d495c8bab89d3295e7dba, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/03ce259e6e1a4d138901f84e30ab32c2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/14a78546eeed4f8e801fce2b195a44d2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/133e7bc3d42541e3b4585088119d3ee1, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/c90342ca322d4d04babc1860628ac216, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/c93230932a5942c0b1206550de361af6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/4aa88f0c171540678212ca49b11a4714] to archive 2024-11-14T00:28:54,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:28:54,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/9a38c8b8998349f7b3f54fac21190878 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/9a38c8b8998349f7b3f54fac21190878 2024-11-14T00:28:54,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/1606cd57cfc24183a3e37c302002947b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/1606cd57cfc24183a3e37c302002947b 2024-11-14T00:28:54,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/52cc026e0c3040bbaec7aaae4797dd01 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/52cc026e0c3040bbaec7aaae4797dd01 2024-11-14T00:28:54,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f7f7cdf3ba2c4f54a2ff0112b7031c41 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f7f7cdf3ba2c4f54a2ff0112b7031c41 2024-11-14T00:28:54,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/ce3153ec72cc42eaadc5a30fda5658c8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/ce3153ec72cc42eaadc5a30fda5658c8 2024-11-14T00:28:54,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/49f23a428adf4cc08728bfae57e3da27 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/49f23a428adf4cc08728bfae57e3da27 2024-11-14T00:28:54,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/2eae2ae581ff4d13bbf4605b6ececa6c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/2eae2ae581ff4d13bbf4605b6ececa6c 2024-11-14T00:28:54,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/8626967a3af64623832bff64143199da to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/8626967a3af64623832bff64143199da 2024-11-14T00:28:54,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/01cfb3e56cce4a00b67128fa6eb3a3f9 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/01cfb3e56cce4a00b67128fa6eb3a3f9 2024-11-14T00:28:54,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/01d9cfaba4b74466bafda523025a66bc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/01d9cfaba4b74466bafda523025a66bc 2024-11-14T00:28:54,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/3f47874e343e49639c7fc4898831a2a0 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/3f47874e343e49639c7fc4898831a2a0 2024-11-14T00:28:54,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/2ad93d65b94940e7be8e4c60707e6bb1 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/2ad93d65b94940e7be8e4c60707e6bb1 2024-11-14T00:28:54,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/ac4d756817ea420c8401a4bc78493611 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/ac4d756817ea420c8401a4bc78493611 2024-11-14T00:28:54,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/6fb69cd9e5ab48ddae083c36d51ad4ff to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/6fb69cd9e5ab48ddae083c36d51ad4ff 2024-11-14T00:28:54,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/aa0e152083684eb184593a4c53fed5b9 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/aa0e152083684eb184593a4c53fed5b9 2024-11-14T00:28:54,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d6ede8630e9b49a5a410279329894dfd to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d6ede8630e9b49a5a410279329894dfd 2024-11-14T00:28:54,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d4c49b76d7c34c678c70add0098ec995 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d4c49b76d7c34c678c70add0098ec995 2024-11-14T00:28:54,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d8128998796541dabde32176ddc20508 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/d8128998796541dabde32176ddc20508 2024-11-14T00:28:54,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/4ff23a46dc0641999464a5d063ad6795 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/4ff23a46dc0641999464a5d063ad6795 2024-11-14T00:28:54,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f7ab36cb533a45ccbf7fe2118fa7d6f8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f7ab36cb533a45ccbf7fe2118fa7d6f8 2024-11-14T00:28:54,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/04d83d35af8e4de381b9f8b33618ab36 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/04d83d35af8e4de381b9f8b33618ab36 2024-11-14T00:28:54,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/7ec80c2aba414e34ade37659cd3f98fb to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/7ec80c2aba414e34ade37659cd3f98fb 2024-11-14T00:28:54,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/5740190e37674a94afa95c53992a2b9d to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/5740190e37674a94afa95c53992a2b9d 2024-11-14T00:28:54,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f1d5b22905394dc09578b3382c02439f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/f1d5b22905394dc09578b3382c02439f 2024-11-14T00:28:54,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/649a2967ef33408f91c62d38503eabd4 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/649a2967ef33408f91c62d38503eabd4 2024-11-14T00:28:54,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/49aafafd6e1d495c8bab89d3295e7dba to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/49aafafd6e1d495c8bab89d3295e7dba 2024-11-14T00:28:54,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/03ce259e6e1a4d138901f84e30ab32c2 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/03ce259e6e1a4d138901f84e30ab32c2 2024-11-14T00:28:54,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/14a78546eeed4f8e801fce2b195a44d2 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/14a78546eeed4f8e801fce2b195a44d2 2024-11-14T00:28:54,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/133e7bc3d42541e3b4585088119d3ee1 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/133e7bc3d42541e3b4585088119d3ee1 2024-11-14T00:28:54,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/c90342ca322d4d04babc1860628ac216 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/c90342ca322d4d04babc1860628ac216 2024-11-14T00:28:54,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/c93230932a5942c0b1206550de361af6 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/c93230932a5942c0b1206550de361af6 2024-11-14T00:28:54,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/4aa88f0c171540678212ca49b11a4714 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/4aa88f0c171540678212ca49b11a4714 2024-11-14T00:28:54,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e83950b6268b4baf8d22c74035583a72, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/53c275da8dc749d8b97888754923af5b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/6c7c1e9a664445c9b8367261aa4123e9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/21b9a8f68dbd42fe891bdd0dae2a3058, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/9c04391a7c134bb6b7094f154c1d230c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/322171c2b4a043798ef9e01588947a39, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/f4eae73f7aa746aab1ef4b22a537d004, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e6d145db5fa640b2b462e8c20e374298, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/8b2fd3f28d5c4a93af51199404766bd6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/217c1d40585d4abaa06f1fdf563cd7c1, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/a33554ed234c42e7ac62f71a664ec6e3, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/545faf0cfce542c0b112fe98963dc30e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/148b7ceb4cdf4a77bf5ed424b27ae01a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/7346dc25b0904ec9b4fecf6b09cd2e1d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c72a609fc0e14994b141f30e27cfeef0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/dabe352cb1bc41c4b4b5b5659aa47f09, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/5dcf7bd4c43f4dbeba6a9c21199eaf2e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/1c1416405ab649c3a7c522eaa3ac18a7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/0c7793094a654e14a5c0e392c8d198e9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/3908c933bf254e4db4432fbf284aed82, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/a346a7f60d4442aeb6e6d91791bca7b8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/04af2697e3974f44a947eefa433eb78e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/ee23e377effb4a1bbe8ae90ddfe2d322, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/ee98b5a34d344f28b69698acae1a5a27, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/d0d7e320f5a34ae8adb582637b035bcb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e390c23d2850421b989f5ee052ca80cc, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/60f0739e137b4969af5c16b36990f8ae, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/5056b6e109ff413ba1df2e590f3bf8bb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c5ce9da8029e49fa83e97b76661e5d79, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c7f729d6e0834aec9905bef21453b27b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/4d58504660dc4c1bab2a81ab037913cb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/6a40bd0e3a244263ab9983838c855dd2] to archive 2024-11-14T00:28:54,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:28:54,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e83950b6268b4baf8d22c74035583a72 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e83950b6268b4baf8d22c74035583a72 2024-11-14T00:28:54,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/53c275da8dc749d8b97888754923af5b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/53c275da8dc749d8b97888754923af5b 2024-11-14T00:28:54,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/6c7c1e9a664445c9b8367261aa4123e9 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/6c7c1e9a664445c9b8367261aa4123e9 2024-11-14T00:28:54,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/21b9a8f68dbd42fe891bdd0dae2a3058 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/21b9a8f68dbd42fe891bdd0dae2a3058 2024-11-14T00:28:54,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/9c04391a7c134bb6b7094f154c1d230c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/9c04391a7c134bb6b7094f154c1d230c 2024-11-14T00:28:54,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/322171c2b4a043798ef9e01588947a39 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/322171c2b4a043798ef9e01588947a39 2024-11-14T00:28:54,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/f4eae73f7aa746aab1ef4b22a537d004 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/f4eae73f7aa746aab1ef4b22a537d004 2024-11-14T00:28:54,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e6d145db5fa640b2b462e8c20e374298 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e6d145db5fa640b2b462e8c20e374298 2024-11-14T00:28:54,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/8b2fd3f28d5c4a93af51199404766bd6 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/8b2fd3f28d5c4a93af51199404766bd6 2024-11-14T00:28:54,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/217c1d40585d4abaa06f1fdf563cd7c1 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/217c1d40585d4abaa06f1fdf563cd7c1 2024-11-14T00:28:54,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/a33554ed234c42e7ac62f71a664ec6e3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/a33554ed234c42e7ac62f71a664ec6e3 2024-11-14T00:28:54,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/545faf0cfce542c0b112fe98963dc30e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/545faf0cfce542c0b112fe98963dc30e 2024-11-14T00:28:54,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/148b7ceb4cdf4a77bf5ed424b27ae01a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/148b7ceb4cdf4a77bf5ed424b27ae01a 2024-11-14T00:28:54,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/7346dc25b0904ec9b4fecf6b09cd2e1d to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/7346dc25b0904ec9b4fecf6b09cd2e1d 2024-11-14T00:28:54,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c72a609fc0e14994b141f30e27cfeef0 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c72a609fc0e14994b141f30e27cfeef0 2024-11-14T00:28:54,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/dabe352cb1bc41c4b4b5b5659aa47f09 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/dabe352cb1bc41c4b4b5b5659aa47f09 2024-11-14T00:28:54,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/5dcf7bd4c43f4dbeba6a9c21199eaf2e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/5dcf7bd4c43f4dbeba6a9c21199eaf2e 2024-11-14T00:28:54,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/1c1416405ab649c3a7c522eaa3ac18a7 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/1c1416405ab649c3a7c522eaa3ac18a7 2024-11-14T00:28:54,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/0c7793094a654e14a5c0e392c8d198e9 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/0c7793094a654e14a5c0e392c8d198e9 2024-11-14T00:28:54,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/3908c933bf254e4db4432fbf284aed82 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/3908c933bf254e4db4432fbf284aed82 2024-11-14T00:28:54,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/a346a7f60d4442aeb6e6d91791bca7b8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/a346a7f60d4442aeb6e6d91791bca7b8 2024-11-14T00:28:54,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/04af2697e3974f44a947eefa433eb78e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/04af2697e3974f44a947eefa433eb78e 2024-11-14T00:28:54,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/ee23e377effb4a1bbe8ae90ddfe2d322 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/ee23e377effb4a1bbe8ae90ddfe2d322 2024-11-14T00:28:54,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/ee98b5a34d344f28b69698acae1a5a27 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/ee98b5a34d344f28b69698acae1a5a27 2024-11-14T00:28:54,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/d0d7e320f5a34ae8adb582637b035bcb to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/d0d7e320f5a34ae8adb582637b035bcb 2024-11-14T00:28:54,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e390c23d2850421b989f5ee052ca80cc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/e390c23d2850421b989f5ee052ca80cc 2024-11-14T00:28:54,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/60f0739e137b4969af5c16b36990f8ae to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/60f0739e137b4969af5c16b36990f8ae 2024-11-14T00:28:54,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/5056b6e109ff413ba1df2e590f3bf8bb to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/5056b6e109ff413ba1df2e590f3bf8bb 2024-11-14T00:28:54,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c5ce9da8029e49fa83e97b76661e5d79 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c5ce9da8029e49fa83e97b76661e5d79 2024-11-14T00:28:54,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c7f729d6e0834aec9905bef21453b27b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/c7f729d6e0834aec9905bef21453b27b 2024-11-14T00:28:54,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/4d58504660dc4c1bab2a81ab037913cb to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/4d58504660dc4c1bab2a81ab037913cb 2024-11-14T00:28:54,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/6a40bd0e3a244263ab9983838c855dd2 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/6a40bd0e3a244263ab9983838c855dd2 2024-11-14T00:28:54,360 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/recovered.edits/472.seqid, newMaxSeqId=472, maxSeqId=1 2024-11-14T00:28:54,362 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. 2024-11-14T00:28:54,362 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for 328b146c51a69a897800000a78ffe3a7: Waiting for close lock at 1731544132898Running coprocessor pre-close hooks at 1731544132898Disabling compacts and flushes for region at 1731544132898Disabling writes for close at 1731544132898Obtaining lock to block concurrent updates at 1731544132898Preparing flush snapshotting stores in 328b146c51a69a897800000a78ffe3a7 at 1731544132898Finished memstore snapshotting TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731544132898Flushing stores of TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7. at 1731544132899 (+1 ms)Flushing 328b146c51a69a897800000a78ffe3a7/A: creating writer at 1731544132899Flushing 328b146c51a69a897800000a78ffe3a7/A: appending metadata at 1731544132909 (+10 ms)Flushing 328b146c51a69a897800000a78ffe3a7/A: closing flushed file at 1731544132909Flushing 328b146c51a69a897800000a78ffe3a7/B: creating writer at 1731544132961 (+52 ms)Flushing 328b146c51a69a897800000a78ffe3a7/B: appending metadata at 1731544132977 (+16 ms)Flushing 328b146c51a69a897800000a78ffe3a7/B: closing flushed file at 1731544132977Flushing 328b146c51a69a897800000a78ffe3a7/C: creating writer at 1731544133426 (+449 ms)Flushing 328b146c51a69a897800000a78ffe3a7/C: appending metadata at 1731544133429 (+3 ms)Flushing 328b146c51a69a897800000a78ffe3a7/C: closing flushed file at 1731544133429Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4de48e82: reopening flushed file at 1731544133848 (+419 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4eb07c66: reopening flushed file at 1731544133854 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58b8cc8b: reopening flushed file at 1731544133862 (+8 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 328b146c51a69a897800000a78ffe3a7 in 972ms, sequenceid=469, compaction requested=false at 1731544133870 (+8 ms)Writing region close event to WAL at 1731544134347 (+477 ms)Running coprocessor post-close hooks at 1731544134361 (+14 ms)Closed at 1731544134361 2024-11-14T00:28:54,377 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed 328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:54,392 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=328b146c51a69a897800000a78ffe3a7, regionState=CLOSED 2024-11-14T00:28:54,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:28:54,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-14T00:28:54,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure 328b146c51a69a897800000a78ffe3a7, server=63baa11b75f3,33199,1731544045387 in 1.6650 sec 2024-11-14T00:28:54,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-11-14T00:28:54,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=328b146c51a69a897800000a78ffe3a7, UNASSIGN in 1.6740 sec 2024-11-14T00:28:54,420 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544134420"}]},"ts":"1731544134420"} 2024-11-14T00:28:54,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-14T00:28:54,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6790 sec 2024-11-14T00:28:54,426 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T00:28:54,426 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T00:28:54,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6980 sec 2024-11-14T00:28:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T00:28:54,885 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:54,886 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-14T00:28:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:54,888 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T00:28:54,891 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=80, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:54,904 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:54,908 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T00:28:54,932 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/recovered.edits] 2024-11-14T00:28:54,975 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/28000251782245989710c5b05e303798 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/28000251782245989710c5b05e303798 2024-11-14T00:28:54,977 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/579f4ed2c89340a7a6ce85a31e14b890 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/A/579f4ed2c89340a7a6ce85a31e14b890 2024-11-14T00:28:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T00:28:55,036 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/387ff9e3f4534fc7885e441ba89d86b0 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/387ff9e3f4534fc7885e441ba89d86b0 2024-11-14T00:28:55,053 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/88fae82f91c74eadb42f4a8fc7ad223a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/B/88fae82f91c74eadb42f4a8fc7ad223a 2024-11-14T00:28:55,063 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/1d1e7d89029e48818319c12f825fa62c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/1d1e7d89029e48818319c12f825fa62c 2024-11-14T00:28:55,064 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/1f56133572664c4b940925fbdc1d3e25 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/C/1f56133572664c4b940925fbdc1d3e25 2024-11-14T00:28:55,074 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/recovered.edits/472.seqid to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7/recovered.edits/472.seqid 2024-11-14T00:28:55,076 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/328b146c51a69a897800000a78ffe3a7 2024-11-14T00:28:55,076 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T00:28:55,082 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=80, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:55,085 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T00:28:55,088 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T00:28:55,090 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=80, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:55,090 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T00:28:55,090 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731544135090"}]},"ts":"9223372036854775807"} 2024-11-14T00:28:55,093 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T00:28:55,093 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 328b146c51a69a897800000a78ffe3a7, NAME => 'TestAcidGuarantees,,1731544107725.328b146c51a69a897800000a78ffe3a7.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T00:28:55,093 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T00:28:55,093 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731544135093"}]},"ts":"9223372036854775807"} 2024-11-14T00:28:55,095 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T00:28:55,097 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=80, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:55,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 211 msec 2024-11-14T00:28:55,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T00:28:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T00:28:55,206 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:55,218 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239 (was 237) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1906822607_22 at /127.0.0.1:38098 [Waiting for operation #386] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1495897995_22 at /127.0.0.1:54016 [Waiting for operation #248] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1190 (was 1210), ProcessCount=11 (was 11), AvailableMemoryMB=3124 (was 1669) - AvailableMemoryMB LEAK? - 2024-11-14T00:28:55,231 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=1190, ProcessCount=11, AvailableMemoryMB=3123 2024-11-14T00:28:55,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T00:28:55,232 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T00:28:55,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:55,234 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T00:28:55,234 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:55,234 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 81 2024-11-14T00:28:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-14T00:28:55,235 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T00:28:55,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742131_1307 (size=959) 2024-11-14T00:28:55,268 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c 2024-11-14T00:28:55,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742132_1308 (size=53) 2024-11-14T00:28:55,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:28:55,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a0f5137c718f1ce8c674c5a78d56c3d3, disabling compactions & flushes 2024-11-14T00:28:55,292 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:55,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:55,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. after waiting 0 ms 2024-11-14T00:28:55,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:55,292 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:55,292 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a0f5137c718f1ce8c674c5a78d56c3d3: Waiting for close lock at 1731544135292Disabling compacts and flushes for region at 1731544135292Disabling writes for close at 1731544135292Writing region close event to WAL at 1731544135292Closed at 1731544135292 2024-11-14T00:28:55,295 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T00:28:55,296 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731544135295"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731544135295"}]},"ts":"1731544135295"} 2024-11-14T00:28:55,298 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T00:28:55,299 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T00:28:55,299 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544135299"}]},"ts":"1731544135299"} 2024-11-14T00:28:55,301 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T00:28:55,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0f5137c718f1ce8c674c5a78d56c3d3, ASSIGN}] 2024-11-14T00:28:55,306 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0f5137c718f1ce8c674c5a78d56c3d3, ASSIGN 2024-11-14T00:28:55,308 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0f5137c718f1ce8c674c5a78d56c3d3, ASSIGN; state=OFFLINE, location=63baa11b75f3,33199,1731544045387; forceNewPlan=false, retain=false 2024-11-14T00:28:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-14T00:28:55,461 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=a0f5137c718f1ce8c674c5a78d56c3d3, regionState=OPENING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:55,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0f5137c718f1ce8c674c5a78d56c3d3, ASSIGN because future has completed 2024-11-14T00:28:55,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:28:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-14T00:28:55,619 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:55,619 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7752): Opening region: {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} 2024-11-14T00:28:55,620 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:55,620 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:28:55,620 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7794): checking encryption for a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:55,620 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7797): checking classloading for a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:55,621 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:55,624 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0f5137c718f1ce8c674c5a78d56c3d3 columnFamilyName A 2024-11-14T00:28:55,625 DEBUG [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:55,628 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] regionserver.HStore(327): Store=a0f5137c718f1ce8c674c5a78d56c3d3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:28:55,629 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:55,632 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0f5137c718f1ce8c674c5a78d56c3d3 columnFamilyName B 2024-11-14T00:28:55,632 DEBUG [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:55,634 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] regionserver.HStore(327): Store=a0f5137c718f1ce8c674c5a78d56c3d3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:28:55,634 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:55,635 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0f5137c718f1ce8c674c5a78d56c3d3 columnFamilyName C 2024-11-14T00:28:55,636 DEBUG [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:55,636 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] regionserver.HStore(327): Store=a0f5137c718f1ce8c674c5a78d56c3d3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:28:55,636 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1038): replaying wal for a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:55,637 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:55,637 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:55,638 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1048): stopping wal replay for a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:55,638 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1060): Cleaning up temporary data for a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:55,639 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T00:28:55,640 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): writing seq id for a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:55,648 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T00:28:55,650 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1114): Opened a0f5137c718f1ce8c674c5a78d56c3d3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69857182, jitterRate=0.040953129529953}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T00:28:55,650 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:55,689 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1006): Region open journal for a0f5137c718f1ce8c674c5a78d56c3d3: Running coprocessor pre-open hook at 1731544135620Writing region info on filesystem at 1731544135620Initializing all the Stores at 1731544135621 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544135621Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544135621Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544135621Cleaning up temporary data from old regions at 1731544135638 (+17 ms)Running coprocessor post-open hooks at 1731544135650 (+12 ms)Region opened successfully at 1731544135689 (+39 ms) 2024-11-14T00:28:55,696 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., pid=83, masterSystemTime=1731544135616 2024-11-14T00:28:55,706 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=a0f5137c718f1ce8c674c5a78d56c3d3, regionState=OPEN, openSeqNum=2, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:55,708 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:55,708 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:55,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:28:55,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-14T00:28:55,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 in 269 msec 2024-11-14T00:28:55,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-14T00:28:55,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0f5137c718f1ce8c674c5a78d56c3d3, ASSIGN in 447 msec 2024-11-14T00:28:55,779 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T00:28:55,779 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544135779"}]},"ts":"1731544135779"} 2024-11-14T00:28:55,790 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T00:28:55,794 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T00:28:55,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 562 msec 2024-11-14T00:28:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-14T00:28:55,865 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:55,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1886fcad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:55,866 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:55,867 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:55,868 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a606ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:55,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:55,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:55,869 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:55,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@427565fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:55,871 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:55,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:55,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:55,873 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:55,874 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T00:28:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 63baa11b75f3,36271,1731544044347 2024-11-14T00:28:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@44246ec1 2024-11-14T00:28:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T00:28:55,876 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T00:28:55,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T00:28:55,881 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T00:28:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T00:28:55,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742133_1309 (size=995) 2024-11-14T00:28:56,307 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-14T00:28:56,307 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-14T00:28:56,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T00:28:56,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0f5137c718f1ce8c674c5a78d56c3d3, REOPEN/MOVE}] 2024-11-14T00:28:56,325 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0f5137c718f1ce8c674c5a78d56c3d3, REOPEN/MOVE 2024-11-14T00:28:56,325 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=a0f5137c718f1ce8c674c5a78d56c3d3, regionState=CLOSING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:56,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0f5137c718f1ce8c674c5a78d56c3d3, REOPEN/MOVE because future has completed 2024-11-14T00:28:56,328 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T00:28:56,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:28:56,481 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:56,481 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T00:28:56,481 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing a0f5137c718f1ce8c674c5a78d56c3d3, disabling compactions & flushes 2024-11-14T00:28:56,481 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:56,481 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:56,481 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. after waiting 0 ms 2024-11-14T00:28:56,481 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:56,502 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T00:28:56,502 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:56,503 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for a0f5137c718f1ce8c674c5a78d56c3d3: Waiting for close lock at 1731544136481Running coprocessor pre-close hooks at 1731544136481Disabling compacts and flushes for region at 1731544136481Disabling writes for close at 1731544136481Writing region close event to WAL at 1731544136488 (+7 ms)Running coprocessor post-close hooks at 1731544136502 (+14 ms)Closed at 1731544136502 2024-11-14T00:28:56,503 WARN [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionServer(3299): Not adding moved region record: a0f5137c718f1ce8c674c5a78d56c3d3 to self. 2024-11-14T00:28:56,506 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=a0f5137c718f1ce8c674c5a78d56c3d3, regionState=CLOSED 2024-11-14T00:28:56,508 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:56,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:28:56,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-14T00:28:56,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 in 183 msec 2024-11-14T00:28:56,515 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0f5137c718f1ce8c674c5a78d56c3d3, REOPEN/MOVE; state=CLOSED, location=63baa11b75f3,33199,1731544045387; forceNewPlan=false, retain=true 2024-11-14T00:28:56,666 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=a0f5137c718f1ce8c674c5a78d56c3d3, regionState=OPENING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:56,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0f5137c718f1ce8c674c5a78d56c3d3, REOPEN/MOVE because future has completed 2024-11-14T00:28:56,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:28:56,824 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:56,824 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} 2024-11-14T00:28:56,825 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:56,825 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:28:56,825 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:56,825 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:56,826 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:56,827 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0f5137c718f1ce8c674c5a78d56c3d3 columnFamilyName A 2024-11-14T00:28:56,830 DEBUG [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:56,831 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] regionserver.HStore(327): Store=a0f5137c718f1ce8c674c5a78d56c3d3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:28:56,832 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:56,833 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0f5137c718f1ce8c674c5a78d56c3d3 columnFamilyName B 2024-11-14T00:28:56,833 DEBUG [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:56,834 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] regionserver.HStore(327): Store=a0f5137c718f1ce8c674c5a78d56c3d3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:28:56,835 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:56,836 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0f5137c718f1ce8c674c5a78d56c3d3 columnFamilyName C 2024-11-14T00:28:56,836 DEBUG [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:56,837 INFO [StoreOpener-a0f5137c718f1ce8c674c5a78d56c3d3-1 {}] regionserver.HStore(327): Store=a0f5137c718f1ce8c674c5a78d56c3d3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:28:56,837 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:56,839 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:56,841 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:56,845 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:56,845 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:56,846 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T00:28:56,849 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:56,851 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened a0f5137c718f1ce8c674c5a78d56c3d3; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59433839, jitterRate=-0.11436678469181061}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T00:28:56,851 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:56,853 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for a0f5137c718f1ce8c674c5a78d56c3d3: Running coprocessor pre-open hook at 1731544136825Writing region info on filesystem at 1731544136825Initializing all the Stores at 1731544136826 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544136826Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544136826Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544136826Cleaning up temporary data from old regions at 1731544136845 (+19 ms)Running coprocessor post-open hooks at 1731544136851 (+6 ms)Region opened successfully at 1731544136852 (+1 ms) 2024-11-14T00:28:56,854 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., pid=88, masterSystemTime=1731544136821 2024-11-14T00:28:56,856 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:56,856 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:56,857 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=a0f5137c718f1ce8c674c5a78d56c3d3, regionState=OPEN, openSeqNum=5, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:56,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:28:56,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=86 2024-11-14T00:28:56,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 in 190 msec 2024-11-14T00:28:56,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-14T00:28:56,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0f5137c718f1ce8c674c5a78d56c3d3, REOPEN/MOVE in 538 msec 2024-11-14T00:28:56,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-14T00:28:56,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 550 msec 2024-11-14T00:28:56,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 986 msec 2024-11-14T00:28:56,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T00:28:56,870 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:56,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@504f3f40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:56,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:56,871 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:56,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:56,872 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25d6d4d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,872 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:56,872 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:56,872 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:56,873 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:56,874 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39d26542, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7696cee8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:56,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:56,877 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:56,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b7d3f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:56,879 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@465a87d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7089ad6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:56,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:56,888 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:56,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@779aaf91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:56,890 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:56,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cea1cf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a405972, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,892 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:56,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:56,893 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea2db32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:56,896 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:56,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ddf5cb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ec6218, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,900 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:56,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:56,901 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:56,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:56,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:56,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77a2533f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:56,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:56,903 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:56,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce0f976, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1398ece7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:56,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:56,906 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503bd0d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:56,907 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:56,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4530a68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd9f4fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,909 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:56,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:56,909 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:56,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:56,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:56,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc1f776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:56,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:56,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:56,911 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:56,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ae8bb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36bba6b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:56,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:56,913 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:56,913 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@162cf5d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:56,915 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:56,916 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc324b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c37c376, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,928 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:56,929 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:56,930 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:56,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:56,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:56,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625e3708, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:56,930 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:56,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:56,932 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:56,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d18d864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f7f69e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:28:56,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:28:56,938 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:28:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:28:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:28:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fdfd9de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:28:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:28:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:28:56,940 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:28:56,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2838e1a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:28:56,941 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:56,941 DEBUG [Thread-1441 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-14T00:28:56,944 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T00:28:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:28:56,947 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:56,947 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:56,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:56,952 DEBUG [Thread-1426 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:56,952 DEBUG [Thread-1428 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:56,952 DEBUG [Thread-1437 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:56,953 DEBUG [Thread-1439 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:56,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:56,954 DEBUG [Thread-1422 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:56,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:56,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:56,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:56,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:56,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:56,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:56,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:28:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:28:56,958 DEBUG [Thread-1430 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:56,958 DEBUG [Thread-1424 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:56,958 DEBUG [Thread-1433 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:28:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:28:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:28:56,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:56,959 DEBUG [Thread-1435 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:28:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:56,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:56,960 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:56,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:56,961 DEBUG [Thread-1435 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:28:56,961 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:56,961 DEBUG [Thread-1435 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:28:56,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:28:56,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:28:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:28:56,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:28:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:28:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:56,974 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:56,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T00:28:57,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114601279a36d154cee8799b103e1dbf281_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_0/A:col10/1731544136960/Put/seqid=0 2024-11-14T00:28:57,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742134_1310 (size=12154) 2024-11-14T00:28:57,030 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:57,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544197023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,036 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114601279a36d154cee8799b103e1dbf281_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114601279a36d154cee8799b103e1dbf281_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:57,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544197024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544197025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544197028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544197029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,039 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/0aacb2c43f2f4adf8cb5d99425a42dcf, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:28:57,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/0aacb2c43f2f4adf8cb5d99425a42dcf is 175, key is test_row_0/A:col10/1731544136960/Put/seqid=0 2024-11-14T00:28:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T00:28:57,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742135_1311 (size=30955) 2024-11-14T00:28:57,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T00:28:57,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:57,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:28:57,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:57,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544197145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544197145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544197146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544197146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544197146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T00:28:57,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:57,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:28:57,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:57,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T00:28:57,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544197355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544197366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544197366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544197366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544197368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T00:28:57,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:57,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:28:57,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:57,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,471 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/0aacb2c43f2f4adf8cb5d99425a42dcf 2024-11-14T00:28:57,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/4a904103772b4f42af20514563936a66 is 50, key is test_row_0/B:col10/1731544136960/Put/seqid=0 2024-11-14T00:28:57,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742136_1312 (size=12001) 2024-11-14T00:28:57,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/4a904103772b4f42af20514563936a66 2024-11-14T00:28:57,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T00:28:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:28:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:57,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T00:28:57,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/0a98f8c868444ee2bb5e6eb0e09cf95f is 50, key is test_row_0/C:col10/1731544136960/Put/seqid=0 2024-11-14T00:28:57,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544197666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544197675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544197675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544197675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544197687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:57,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742137_1313 (size=12001) 2024-11-14T00:28:57,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T00:28:57,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:57,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:28:57,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:57,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T00:28:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:28:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:57,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:57,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:58,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T00:28:58,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:58,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:28:58,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:58,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:58,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:58,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:58,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T00:28:58,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/0a98f8c868444ee2bb5e6eb0e09cf95f 2024-11-14T00:28:58,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/0aacb2c43f2f4adf8cb5d99425a42dcf as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/0aacb2c43f2f4adf8cb5d99425a42dcf 2024-11-14T00:28:58,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544198186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:58,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/0aacb2c43f2f4adf8cb5d99425a42dcf, entries=150, sequenceid=16, filesize=30.2 K 2024-11-14T00:28:58,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:58,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544198198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:58,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544198199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:58,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544198199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:58,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/4a904103772b4f42af20514563936a66 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/4a904103772b4f42af20514563936a66 2024-11-14T00:28:58,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:58,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:58,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:58,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544198205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:58,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:58,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T00:28:58,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:58,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:28:58,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:58,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:58,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:58,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:58,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/4a904103772b4f42af20514563936a66, entries=150, sequenceid=16, filesize=11.7 K 2024-11-14T00:28:58,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/0a98f8c868444ee2bb5e6eb0e09cf95f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/0a98f8c868444ee2bb5e6eb0e09cf95f 2024-11-14T00:28:58,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/0a98f8c868444ee2bb5e6eb0e09cf95f, entries=150, sequenceid=16, filesize=11.7 K 2024-11-14T00:28:58,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a0f5137c718f1ce8c674c5a78d56c3d3 in 1262ms, sequenceid=16, compaction requested=false 2024-11-14T00:28:58,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:28:58,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-14T00:28:58,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:58,373 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T00:28:58,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f15cd7027311490fa4effb3675cf9a6d_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_0/A:col10/1731544137023/Put/seqid=0 2024-11-14T00:28:58,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742138_1314 (size=12154) 2024-11-14T00:28:58,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:58,849 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f15cd7027311490fa4effb3675cf9a6d_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f15cd7027311490fa4effb3675cf9a6d_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:58,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/edf2b44be63a447f973ccf1242f7bf3e, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:28:58,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/edf2b44be63a447f973ccf1242f7bf3e is 175, key is test_row_0/A:col10/1731544137023/Put/seqid=0 2024-11-14T00:28:58,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742139_1315 (size=30955) 2024-11-14T00:28:58,885 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/edf2b44be63a447f973ccf1242f7bf3e 2024-11-14T00:28:58,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/fab0f924b3454995966f5784a9f7d29d is 50, key is test_row_0/B:col10/1731544137023/Put/seqid=0 2024-11-14T00:28:58,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742140_1316 (size=12001) 2024-11-14T00:28:58,945 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/fab0f924b3454995966f5784a9f7d29d 2024-11-14T00:28:58,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/dd6c27d5365e4f8e86bc36d1cfb030e2 is 50, key is test_row_0/C:col10/1731544137023/Put/seqid=0 2024-11-14T00:28:58,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742141_1317 (size=12001) 2024-11-14T00:28:59,005 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/dd6c27d5365e4f8e86bc36d1cfb030e2 2024-11-14T00:28:59,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/edf2b44be63a447f973ccf1242f7bf3e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/edf2b44be63a447f973ccf1242f7bf3e 2024-11-14T00:28:59,041 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/edf2b44be63a447f973ccf1242f7bf3e, entries=150, sequenceid=40, filesize=30.2 K 2024-11-14T00:28:59,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/fab0f924b3454995966f5784a9f7d29d as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/fab0f924b3454995966f5784a9f7d29d 2024-11-14T00:28:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,059 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/fab0f924b3454995966f5784a9f7d29d, entries=150, sequenceid=40, filesize=11.7 K 2024-11-14T00:28:59,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/dd6c27d5365e4f8e86bc36d1cfb030e2 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/dd6c27d5365e4f8e86bc36d1cfb030e2 2024-11-14T00:28:59,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,079 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/dd6c27d5365e4f8e86bc36d1cfb030e2, entries=150, sequenceid=40, filesize=11.7 K 2024-11-14T00:28:59,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,081 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for a0f5137c718f1ce8c674c5a78d56c3d3 in 708ms, sequenceid=40, compaction requested=false 2024-11-14T00:28:59,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:28:59,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:59,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-14T00:28:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-14T00:28:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-14T00:28:59,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1380 sec 2024-11-14T00:28:59,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 2.1500 sec 2024-11-14T00:28:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-14T00:28:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,105 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:59,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:59,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-14T00:28:59,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,108 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T00:28:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,109 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:59,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T00:28:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-14T00:28:59,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:59,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:28:59,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:59,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-14T00:28:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-14T00:28:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-14T00:28:59,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-14T00:28:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 164 msec 2024-11-14T00:28:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:59,337 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:28:59,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T00:28:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f2b6b71149cb4dd9954cb7145a933e40_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_0/A:col10/1731544139305/Put/seqid=0 2024-11-14T00:28:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-14T00:28:59,425 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:28:59,426 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:28:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742142_1318 (size=24358) 2024-11-14T00:28:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-14T00:28:59,427 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T00:28:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,432 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f2b6b71149cb4dd9954cb7145a933e40_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f2b6b71149cb4dd9954cb7145a933e40_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:28:59,432 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:28:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,433 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/12e02e0ee13f4cbfbee2fe9709b3ce54, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:28:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,437 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:28:59,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:28:59,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/12e02e0ee13f4cbfbee2fe9709b3ce54 is 175, key is test_row_0/A:col10/1731544139305/Put/seqid=0 2024-11-14T00:28:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742143_1319 (size=73995) 2024-11-14T00:28:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,445 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/12e02e0ee13f4cbfbee2fe9709b3ce54 2024-11-14T00:28:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/f27b2bbb2c3a4b5bb04590544b5b1e1c is 50, key is test_row_0/B:col10/1731544139305/Put/seqid=0 2024-11-14T00:28:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742144_1320 (size=23705) 2024-11-14T00:28:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T00:28:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544199519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:59,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:28:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544199536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:59,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:59,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544199538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:59,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:59,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544199541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:59,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:59,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544199540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:59,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:59,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T00:28:59,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:59,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:28:59,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:59,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:59,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:59,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544199656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:59,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:59,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:59,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544199677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:59,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544199677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:59,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:59,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544199677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:59,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:59,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544199677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T00:28:59,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T00:28:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:28:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:59,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:59,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544199875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:59,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T00:28:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:28:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:28:59,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:59,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:59,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:59,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/f27b2bbb2c3a4b5bb04590544b5b1e1c 2024-11-14T00:28:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544199895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544199895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:59,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:59,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:59,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544199895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:59,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:28:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544199896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:28:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:28:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:59,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:28:59,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:28:59,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/62d210dacaca41b0b8752beeff78b378 is 50, key is test_row_0/C:col10/1731544139305/Put/seqid=0 2024-11-14T00:28:59,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742145_1321 (size=12001) 2024-11-14T00:28:59,922 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T00:29:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T00:29:00,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T00:29:00,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:00,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:00,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:00,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:00,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:00,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544200185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:00,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:00,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T00:29:00,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:00,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:00,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:00,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:00,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:00,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544200216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:00,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544200216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:00,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:00,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544200216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:00,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:00,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:00,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544200216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:00,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/62d210dacaca41b0b8752beeff78b378 2024-11-14T00:29:00,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/12e02e0ee13f4cbfbee2fe9709b3ce54 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/12e02e0ee13f4cbfbee2fe9709b3ce54 2024-11-14T00:29:00,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/12e02e0ee13f4cbfbee2fe9709b3ce54, entries=400, sequenceid=52, filesize=72.3 K 2024-11-14T00:29:00,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/f27b2bbb2c3a4b5bb04590544b5b1e1c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/f27b2bbb2c3a4b5bb04590544b5b1e1c 2024-11-14T00:29:00,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/f27b2bbb2c3a4b5bb04590544b5b1e1c, entries=400, sequenceid=52, filesize=23.1 K 2024-11-14T00:29:00,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/62d210dacaca41b0b8752beeff78b378 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/62d210dacaca41b0b8752beeff78b378 2024-11-14T00:29:00,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T00:29:00,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:00,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:00,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:00,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:00,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:00,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/62d210dacaca41b0b8752beeff78b378, entries=150, sequenceid=52, filesize=11.7 K 2024-11-14T00:29:00,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a0f5137c718f1ce8c674c5a78d56c3d3 in 1032ms, sequenceid=52, compaction requested=true 2024-11-14T00:29:00,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:00,370 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:00,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:00,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:00,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:00,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:00,370 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:00,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:00,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:00,371 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135905 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:00,371 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47707 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:00,371 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/A is initiating minor compaction (all files) 2024-11-14T00:29:00,371 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/B is initiating minor compaction (all files) 2024-11-14T00:29:00,371 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/A in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:00,371 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/B in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:00,371 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/4a904103772b4f42af20514563936a66, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/fab0f924b3454995966f5784a9f7d29d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/f27b2bbb2c3a4b5bb04590544b5b1e1c] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=46.6 K 2024-11-14T00:29:00,371 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/0aacb2c43f2f4adf8cb5d99425a42dcf, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/edf2b44be63a447f973ccf1242f7bf3e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/12e02e0ee13f4cbfbee2fe9709b3ce54] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=132.7 K 2024-11-14T00:29:00,371 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:00,371 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/0aacb2c43f2f4adf8cb5d99425a42dcf, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/edf2b44be63a447f973ccf1242f7bf3e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/12e02e0ee13f4cbfbee2fe9709b3ce54] 2024-11-14T00:29:00,372 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aacb2c43f2f4adf8cb5d99425a42dcf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731544136960 2024-11-14T00:29:00,372 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a904103772b4f42af20514563936a66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731544136960 2024-11-14T00:29:00,372 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting edf2b44be63a447f973ccf1242f7bf3e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731544137023 2024-11-14T00:29:00,372 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting fab0f924b3454995966f5784a9f7d29d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731544137023 2024-11-14T00:29:00,372 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12e02e0ee13f4cbfbee2fe9709b3ce54, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544139294 2024-11-14T00:29:00,373 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting f27b2bbb2c3a4b5bb04590544b5b1e1c, keycount=400, bloomtype=ROW, size=23.1 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544139294 2024-11-14T00:29:00,390 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:00,404 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114b542d4eb763c4da280c9ab73a2eaeda5_a0f5137c718f1ce8c674c5a78d56c3d3 store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:00,407 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#B#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:00,407 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/f984b5c67c98406b9320d64d47600909 is 50, key is test_row_0/B:col10/1731544139305/Put/seqid=0 2024-11-14T00:29:00,414 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114b542d4eb763c4da280c9ab73a2eaeda5_a0f5137c718f1ce8c674c5a78d56c3d3, store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:00,414 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b542d4eb763c4da280c9ab73a2eaeda5_a0f5137c718f1ce8c674c5a78d56c3d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:00,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742146_1322 (size=12104) 2024-11-14T00:29:00,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742147_1323 (size=4469) 2024-11-14T00:29:00,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T00:29:00,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:00,521 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T00:29:00,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c21ab6af74f6417983b73fbe313814a7_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_0/A:col10/1731544139533/Put/seqid=0 2024-11-14T00:29:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T00:29:00,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742148_1324 (size=12154) 2024-11-14T00:29:00,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:00,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544200782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544200784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:00,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544200785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:00,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544200785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:00,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:00,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544200787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:00,914 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#A#compaction#273 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:00,915 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/c295b303296c4e0c894016d8aec16b57 is 175, key is test_row_0/A:col10/1731544139305/Put/seqid=0 2024-11-14T00:29:00,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544200906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:00,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544200907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:00,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544200909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:00,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544200909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:00,922 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/f984b5c67c98406b9320d64d47600909 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/f984b5c67c98406b9320d64d47600909 2024-11-14T00:29:00,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544200913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:00,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:00,929 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/B of a0f5137c718f1ce8c674c5a78d56c3d3 into f984b5c67c98406b9320d64d47600909(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:00,929 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:00,929 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/B, priority=13, startTime=1731544140370; duration=0sec 2024-11-14T00:29:00,929 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:00,929 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:B 2024-11-14T00:29:00,930 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:00,932 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:00,932 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/C is initiating minor compaction (all files) 2024-11-14T00:29:00,932 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/C in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:00,932 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/0a98f8c868444ee2bb5e6eb0e09cf95f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/dd6c27d5365e4f8e86bc36d1cfb030e2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/62d210dacaca41b0b8752beeff78b378] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=35.2 K 2024-11-14T00:29:00,932 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a98f8c868444ee2bb5e6eb0e09cf95f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731544136960 2024-11-14T00:29:00,933 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting dd6c27d5365e4f8e86bc36d1cfb030e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731544137023 2024-11-14T00:29:00,933 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 62d210dacaca41b0b8752beeff78b378, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544139305 2024-11-14T00:29:00,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742149_1325 (size=31058) 2024-11-14T00:29:00,962 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#C#compaction#276 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:00,962 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/71394d64981a4cc3a3486ecfe1ec42f0 is 50, key is test_row_0/C:col10/1731544139305/Put/seqid=0 2024-11-14T00:29:00,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742150_1326 (size=12104) 2024-11-14T00:29:00,988 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/71394d64981a4cc3a3486ecfe1ec42f0 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/71394d64981a4cc3a3486ecfe1ec42f0 2024-11-14T00:29:01,001 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/C of a0f5137c718f1ce8c674c5a78d56c3d3 into 71394d64981a4cc3a3486ecfe1ec42f0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:01,001 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:01,001 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/C, priority=13, startTime=1731544140370; duration=0sec 2024-11-14T00:29:01,001 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:01,001 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:C 2024-11-14T00:29:01,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:01,013 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c21ab6af74f6417983b73fbe313814a7_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c21ab6af74f6417983b73fbe313814a7_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:01,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/9c6f6257c48541a1882f850863b8afa6, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:01,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/9c6f6257c48541a1882f850863b8afa6 is 175, key is test_row_0/A:col10/1731544139533/Put/seqid=0 2024-11-14T00:29:01,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742151_1327 (size=30955) 2024-11-14T00:29:01,061 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/9c6f6257c48541a1882f850863b8afa6 2024-11-14T00:29:01,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/576f2aa43fb84120b358a58b3d4b08e0 is 50, key is test_row_0/B:col10/1731544139533/Put/seqid=0 2024-11-14T00:29:01,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742152_1328 (size=12001) 2024-11-14T00:29:01,085 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/576f2aa43fb84120b358a58b3d4b08e0 2024-11-14T00:29:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/681bdf8289b5420391cc031e79a981fc is 50, key is test_row_0/C:col10/1731544139533/Put/seqid=0 2024-11-14T00:29:01,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742153_1329 (size=12001) 2024-11-14T00:29:01,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544201126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:01,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544201126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:01,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544201126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:01,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544201136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:01,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544201136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:01,357 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/c295b303296c4e0c894016d8aec16b57 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/c295b303296c4e0c894016d8aec16b57 2024-11-14T00:29:01,373 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/A of a0f5137c718f1ce8c674c5a78d56c3d3 into c295b303296c4e0c894016d8aec16b57(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-14T00:29:01,373 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:01,373 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/A, priority=13, startTime=1731544140370; duration=1sec 2024-11-14T00:29:01,373 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:01,373 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:A 2024-11-14T00:29:01,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544201445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:01,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:01,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544201456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544201456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:01,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:01,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544201456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544201456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:01,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:01,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:01,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:01,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:01,473 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:01,507 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/681bdf8289b5420391cc031e79a981fc 2024-11-14T00:29:01,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/9c6f6257c48541a1882f850863b8afa6 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/9c6f6257c48541a1882f850863b8afa6 2024-11-14T00:29:01,521 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/9c6f6257c48541a1882f850863b8afa6, entries=150, sequenceid=77, filesize=30.2 K 2024-11-14T00:29:01,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/576f2aa43fb84120b358a58b3d4b08e0 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/576f2aa43fb84120b358a58b3d4b08e0 2024-11-14T00:29:01,531 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/576f2aa43fb84120b358a58b3d4b08e0, entries=150, sequenceid=77, filesize=11.7 K 2024-11-14T00:29:01,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/681bdf8289b5420391cc031e79a981fc as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/681bdf8289b5420391cc031e79a981fc 2024-11-14T00:29:01,537 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/681bdf8289b5420391cc031e79a981fc, entries=150, sequenceid=77, filesize=11.7 K 2024-11-14T00:29:01,538 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a0f5137c718f1ce8c674c5a78d56c3d3 in 1017ms, sequenceid=77, compaction requested=false 2024-11-14T00:29:01,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:01,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:01,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-14T00:29:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-14T00:29:01,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-14T00:29:01,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1050 sec 2024-11-14T00:29:01,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.1200 sec 2024-11-14T00:29:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T00:29:01,575 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:01,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:29:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-14T00:29:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T00:29:01,580 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:29:01,583 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:29:01,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:29:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T00:29:01,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T00:29:01,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:01,745 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T00:29:01,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145e3309f4aabd4e2ca041dd3fa43f4140_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_0/A:col10/1731544140783/Put/seqid=0 2024-11-14T00:29:01,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742154_1330 (size=12154) 2024-11-14T00:29:01,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T00:29:01,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:01,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:02,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544202034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544202037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544202038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544202039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544202040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544202156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544202157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544202167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544202167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544202168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T00:29:02,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:02,212 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145e3309f4aabd4e2ca041dd3fa43f4140_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145e3309f4aabd4e2ca041dd3fa43f4140_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/7b1f79c675d84bdf8dd8e7ea98ebbb26, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/7b1f79c675d84bdf8dd8e7ea98ebbb26 is 175, key is test_row_0/A:col10/1731544140783/Put/seqid=0 2024-11-14T00:29:02,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742155_1331 (size=30955) 2024-11-14T00:29:02,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544202365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544202375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544202375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544202385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544202385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,629 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/7b1f79c675d84bdf8dd8e7ea98ebbb26 2024-11-14T00:29:02,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/a5089d9d21ae437c932f019bdf603af0 is 50, key is test_row_0/B:col10/1731544140783/Put/seqid=0 2024-11-14T00:29:02,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742156_1332 (size=12001) 2024-11-14T00:29:02,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544202676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544202685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544202687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544202699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544202700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:02,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:02,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:02,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T00:29:03,054 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/a5089d9d21ae437c932f019bdf603af0 2024-11-14T00:29:03,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/2428ba8b9bca45fc8a628208cfa29c42 is 50, key is test_row_0/C:col10/1731544140783/Put/seqid=0 2024-11-14T00:29:03,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742157_1333 (size=12001) 2024-11-14T00:29:03,070 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/2428ba8b9bca45fc8a628208cfa29c42 2024-11-14T00:29:03,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/7b1f79c675d84bdf8dd8e7ea98ebbb26 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/7b1f79c675d84bdf8dd8e7ea98ebbb26 2024-11-14T00:29:03,081 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/7b1f79c675d84bdf8dd8e7ea98ebbb26, entries=150, sequenceid=91, filesize=30.2 K 2024-11-14T00:29:03,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/a5089d9d21ae437c932f019bdf603af0 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/a5089d9d21ae437c932f019bdf603af0 2024-11-14T00:29:03,088 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/a5089d9d21ae437c932f019bdf603af0, entries=150, sequenceid=91, filesize=11.7 K 2024-11-14T00:29:03,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/2428ba8b9bca45fc8a628208cfa29c42 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/2428ba8b9bca45fc8a628208cfa29c42 2024-11-14T00:29:03,096 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/2428ba8b9bca45fc8a628208cfa29c42, entries=150, sequenceid=91, filesize=11.7 K 2024-11-14T00:29:03,098 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a0f5137c718f1ce8c674c5a78d56c3d3 in 1353ms, sequenceid=91, compaction requested=true 2024-11-14T00:29:03,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:03,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:03,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-14T00:29:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-14T00:29:03,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-14T00:29:03,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5170 sec 2024-11-14T00:29:03,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.5260 sec 2024-11-14T00:29:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:03,235 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:03,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-14T00:29:03,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114338132fb0680427c821e07350399e477_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_0/A:col10/1731544143217/Put/seqid=0 2024-11-14T00:29:03,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742158_1334 (size=12154) 2024-11-14T00:29:03,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544203237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:03,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:03,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544203251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544203252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:03,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544203253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:03,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544203254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:03,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:03,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:03,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:03,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544203365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:03,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:03,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544203366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:03,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:03,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544203366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:03,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544203376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:03,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544203376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:03,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:03,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:03,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544203585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:03,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544203585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:03,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544203587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:03,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544203595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:03,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544203595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:03,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:03,650 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:03,663 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114338132fb0680427c821e07350399e477_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114338132fb0680427c821e07350399e477_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:03,668 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/bff1406fa7f94e42bc8c84fd6a1d7a65, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:03,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/bff1406fa7f94e42bc8c84fd6a1d7a65 is 175, key is test_row_0/A:col10/1731544143217/Put/seqid=0 2024-11-14T00:29:03,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742159_1335 (size=30955) 2024-11-14T00:29:03,683 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/bff1406fa7f94e42bc8c84fd6a1d7a65 2024-11-14T00:29:03,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/92cf16f4b2cf4f26adc0a53fa995255d is 50, key is test_row_0/B:col10/1731544143217/Put/seqid=0 2024-11-14T00:29:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T00:29:03,726 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:03,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:29:03,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742160_1336 (size=12001) 2024-11-14T00:29:03,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-14T00:29:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T00:29:03,729 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:29:03,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/92cf16f4b2cf4f26adc0a53fa995255d 2024-11-14T00:29:03,729 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:29:03,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:29:03,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/976a697a75d74a9998eea6e2c90c6499 is 50, key is test_row_0/C:col10/1731544143217/Put/seqid=0 2024-11-14T00:29:03,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742161_1337 (size=12001) 2024-11-14T00:29:03,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/976a697a75d74a9998eea6e2c90c6499 2024-11-14T00:29:03,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/bff1406fa7f94e42bc8c84fd6a1d7a65 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bff1406fa7f94e42bc8c84fd6a1d7a65 2024-11-14T00:29:03,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bff1406fa7f94e42bc8c84fd6a1d7a65, entries=150, sequenceid=119, filesize=30.2 K 2024-11-14T00:29:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T00:29:03,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/92cf16f4b2cf4f26adc0a53fa995255d as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/92cf16f4b2cf4f26adc0a53fa995255d 2024-11-14T00:29:03,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/92cf16f4b2cf4f26adc0a53fa995255d, entries=150, sequenceid=119, filesize=11.7 K 2024-11-14T00:29:03,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/976a697a75d74a9998eea6e2c90c6499 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/976a697a75d74a9998eea6e2c90c6499 2024-11-14T00:29:03,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/976a697a75d74a9998eea6e2c90c6499, entries=150, sequenceid=119, filesize=11.7 K 2024-11-14T00:29:03,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for a0f5137c718f1ce8c674c5a78d56c3d3 in 620ms, sequenceid=119, compaction requested=true 2024-11-14T00:29:03,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:03,855 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:29:03,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:03,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:03,855 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:29:03,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:03,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:03,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:03,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:03,859 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:29:03,859 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123923 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:29:03,859 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/B is initiating minor compaction (all files) 2024-11-14T00:29:03,859 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/A is initiating minor compaction (all files) 2024-11-14T00:29:03,859 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/B in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:03,859 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/A in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:03,859 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/f984b5c67c98406b9320d64d47600909, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/576f2aa43fb84120b358a58b3d4b08e0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/a5089d9d21ae437c932f019bdf603af0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/92cf16f4b2cf4f26adc0a53fa995255d] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=47.0 K 2024-11-14T00:29:03,859 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/c295b303296c4e0c894016d8aec16b57, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/9c6f6257c48541a1882f850863b8afa6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/7b1f79c675d84bdf8dd8e7ea98ebbb26, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bff1406fa7f94e42bc8c84fd6a1d7a65] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=121.0 K 2024-11-14T00:29:03,860 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:03,860 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/c295b303296c4e0c894016d8aec16b57, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/9c6f6257c48541a1882f850863b8afa6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/7b1f79c675d84bdf8dd8e7ea98ebbb26, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bff1406fa7f94e42bc8c84fd6a1d7a65] 2024-11-14T00:29:03,860 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting f984b5c67c98406b9320d64d47600909, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544139305 2024-11-14T00:29:03,861 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 576f2aa43fb84120b358a58b3d4b08e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731544139533 2024-11-14T00:29:03,861 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting c295b303296c4e0c894016d8aec16b57, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544139305 2024-11-14T00:29:03,862 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting a5089d9d21ae437c932f019bdf603af0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731544140736 2024-11-14T00:29:03,862 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c6f6257c48541a1882f850863b8afa6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731544139533 2024-11-14T00:29:03,862 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b1f79c675d84bdf8dd8e7ea98ebbb26, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731544140736 2024-11-14T00:29:03,862 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 92cf16f4b2cf4f26adc0a53fa995255d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731544142032 2024-11-14T00:29:03,866 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting bff1406fa7f94e42bc8c84fd6a1d7a65, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731544142032 2024-11-14T00:29:03,880 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#B#compaction#285 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:03,880 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/20f3a0b2734b45c4bd04492bd0496159 is 50, key is test_row_0/B:col10/1731544143217/Put/seqid=0 2024-11-14T00:29:03,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T00:29:03,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:03,886 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-14T00:29:03,893 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:03,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:03,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:03,919 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111421e4daf10cec466d921161e695ceb4a1_a0f5137c718f1ce8c674c5a78d56c3d3 store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:03,922 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111421e4daf10cec466d921161e695ceb4a1_a0f5137c718f1ce8c674c5a78d56c3d3, store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:03,922 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111421e4daf10cec466d921161e695ceb4a1_a0f5137c718f1ce8c674c5a78d56c3d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:03,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742162_1338 (size=12241) 2024-11-14T00:29:03,937 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/20f3a0b2734b45c4bd04492bd0496159 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/20f3a0b2734b45c4bd04492bd0496159 2024-11-14T00:29:03,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114eb6a0dc3caf64ffd9a12f33f8396f011_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_0/A:col10/1731544143253/Put/seqid=0 2024-11-14T00:29:03,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742163_1339 (size=4469) 2024-11-14T00:29:03,943 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/B of a0f5137c718f1ce8c674c5a78d56c3d3 into 20f3a0b2734b45c4bd04492bd0496159(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:03,943 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:03,943 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/B, priority=12, startTime=1731544143855; duration=0sec 2024-11-14T00:29:03,943 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:03,943 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:B 2024-11-14T00:29:03,943 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:29:03,949 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#A#compaction#286 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:03,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742164_1340 (size=12154) 2024-11-14T00:29:03,949 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/9b13fe17f74d4cbf88274c015f7091fd is 175, key is test_row_0/A:col10/1731544143217/Put/seqid=0 2024-11-14T00:29:03,952 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:29:03,952 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/C is initiating minor compaction (all files) 2024-11-14T00:29:03,952 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/C in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:03,953 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/71394d64981a4cc3a3486ecfe1ec42f0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/681bdf8289b5420391cc031e79a981fc, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/2428ba8b9bca45fc8a628208cfa29c42, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/976a697a75d74a9998eea6e2c90c6499] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=47.0 K 2024-11-14T00:29:03,953 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 71394d64981a4cc3a3486ecfe1ec42f0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544139305 2024-11-14T00:29:03,953 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 681bdf8289b5420391cc031e79a981fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731544139533 2024-11-14T00:29:03,954 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 2428ba8b9bca45fc8a628208cfa29c42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731544140736 2024-11-14T00:29:03,954 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 976a697a75d74a9998eea6e2c90c6499, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731544142032 2024-11-14T00:29:03,963 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#C#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:03,963 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/a839975e18244716b299abd5cc3ca768 is 50, key is test_row_0/C:col10/1731544143217/Put/seqid=0 2024-11-14T00:29:03,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742165_1341 (size=31195) 2024-11-14T00:29:03,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742166_1342 (size=12241) 2024-11-14T00:29:04,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544204000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544203999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544204000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544204013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544204014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T00:29:04,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544204126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544204126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544204129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544204135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544204135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544204345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544204347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544204347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,355 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114eb6a0dc3caf64ffd9a12f33f8396f011_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb6a0dc3caf64ffd9a12f33f8396f011_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T00:29:04,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/cd6b2eb2d6e0451fb286f15202effb04, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:04,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/cd6b2eb2d6e0451fb286f15202effb04 is 175, key is test_row_0/A:col10/1731544143253/Put/seqid=0 2024-11-14T00:29:04,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544204355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544204355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742167_1343 (size=30955) 2024-11-14T00:29:04,404 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/9b13fe17f74d4cbf88274c015f7091fd as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/9b13fe17f74d4cbf88274c015f7091fd 2024-11-14T00:29:04,405 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/a839975e18244716b299abd5cc3ca768 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a839975e18244716b299abd5cc3ca768 2024-11-14T00:29:04,415 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/A of a0f5137c718f1ce8c674c5a78d56c3d3 into 9b13fe17f74d4cbf88274c015f7091fd(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:04,415 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/C of a0f5137c718f1ce8c674c5a78d56c3d3 into a839975e18244716b299abd5cc3ca768(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:04,415 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:04,415 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:04,415 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/A, priority=12, startTime=1731544143855; duration=0sec 2024-11-14T00:29:04,415 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/C, priority=12, startTime=1731544143855; duration=0sec 2024-11-14T00:29:04,415 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:04,415 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:A 2024-11-14T00:29:04,415 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:04,415 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:C 2024-11-14T00:29:04,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544204656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544204659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544204665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544204675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544204676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:04,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:04,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:04,788 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/cd6b2eb2d6e0451fb286f15202effb04 2024-11-14T00:29:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/3579ab65e5db427ea88d16f6f8b4bdd2 is 50, key is test_row_0/B:col10/1731544143253/Put/seqid=0 2024-11-14T00:29:04,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T00:29:04,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742168_1344 (size=12001) 2024-11-14T00:29:04,908 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T00:29:04,908 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T00:29:05,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544205175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:05,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544205186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:05,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544205188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:05,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544205195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:05,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:05,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544205209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:05,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:05,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:05,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:05,278 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/3579ab65e5db427ea88d16f6f8b4bdd2 2024-11-14T00:29:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/cc5b90f33bdd4e78af11adef0c5266e7 is 50, key is test_row_0/C:col10/1731544143253/Put/seqid=0 2024-11-14T00:29:05,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742169_1345 (size=12001) 2024-11-14T00:29:05,345 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/cc5b90f33bdd4e78af11adef0c5266e7 2024-11-14T00:29:05,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/cd6b2eb2d6e0451fb286f15202effb04 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/cd6b2eb2d6e0451fb286f15202effb04 2024-11-14T00:29:05,374 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/cd6b2eb2d6e0451fb286f15202effb04, entries=150, sequenceid=127, filesize=30.2 K 2024-11-14T00:29:05,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/3579ab65e5db427ea88d16f6f8b4bdd2 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/3579ab65e5db427ea88d16f6f8b4bdd2 2024-11-14T00:29:05,386 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/3579ab65e5db427ea88d16f6f8b4bdd2, entries=150, sequenceid=127, filesize=11.7 K 2024-11-14T00:29:05,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/cc5b90f33bdd4e78af11adef0c5266e7 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/cc5b90f33bdd4e78af11adef0c5266e7 2024-11-14T00:29:05,406 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/cc5b90f33bdd4e78af11adef0c5266e7, entries=150, sequenceid=127, filesize=11.7 K 2024-11-14T00:29:05,407 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for a0f5137c718f1ce8c674c5a78d56c3d3 in 1521ms, sequenceid=127, compaction requested=false 2024-11-14T00:29:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:05,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-14T00:29:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-14T00:29:05,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-14T00:29:05,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6800 sec 2024-11-14T00:29:05,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.6930 sec 2024-11-14T00:29:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T00:29:05,875 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:05,876 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:29:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-14T00:29:05,878 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:29:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T00:29:05,880 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:29:05,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:29:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T00:29:06,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T00:29:06,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:06,058 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-14T00:29:06,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cb9d49c9d518450f962041e32bc16a43_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_0/A:col10/1731544143998/Put/seqid=0 2024-11-14T00:29:06,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742170_1346 (size=12304) 2024-11-14T00:29:06,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T00:29:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:06,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:06,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544206215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544206219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:06,229 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_1' in a0f5137c718f1ce8c674c5a78d56c3d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:06,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:06,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544206229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:06,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544206229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:06,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:06,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544206236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:06,244 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_1' in a0f5137c718f1ce8c674c5a78d56c3d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:06,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544206336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:06,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544206348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:06,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544206348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:06,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T00:29:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:06,550 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cb9d49c9d518450f962041e32bc16a43_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cb9d49c9d518450f962041e32bc16a43_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:06,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/8fc5275223eb4688af454ed20d47ed55, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:06,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/8fc5275223eb4688af454ed20d47ed55 is 175, key is test_row_0/A:col10/1731544143998/Put/seqid=0 2024-11-14T00:29:06,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544206556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:06,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544206566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544206566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:06,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742171_1347 (size=31105) 2024-11-14T00:29:06,597 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/8fc5275223eb4688af454ed20d47ed55 2024-11-14T00:29:06,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/6df26c50bc9b45a686534f80204b38ca is 50, key is test_row_0/B:col10/1731544143998/Put/seqid=0 2024-11-14T00:29:06,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742172_1348 (size=12151) 2024-11-14T00:29:06,678 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/6df26c50bc9b45a686534f80204b38ca 2024-11-14T00:29:06,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/c526db6f9e5b4ad0be935f979ca3c48a is 50, key is test_row_0/C:col10/1731544143998/Put/seqid=0 2024-11-14T00:29:06,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742173_1349 (size=12151) 2024-11-14T00:29:06,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544206890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:06,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544206892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:06,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:06,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544206892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T00:29:07,129 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/c526db6f9e5b4ad0be935f979ca3c48a 2024-11-14T00:29:07,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/8fc5275223eb4688af454ed20d47ed55 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/8fc5275223eb4688af454ed20d47ed55 2024-11-14T00:29:07,178 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/8fc5275223eb4688af454ed20d47ed55, entries=150, sequenceid=159, filesize=30.4 K 2024-11-14T00:29:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/6df26c50bc9b45a686534f80204b38ca as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/6df26c50bc9b45a686534f80204b38ca 2024-11-14T00:29:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,215 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/6df26c50bc9b45a686534f80204b38ca, entries=150, sequenceid=159, filesize=11.9 K 2024-11-14T00:29:07,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/c526db6f9e5b4ad0be935f979ca3c48a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/c526db6f9e5b4ad0be935f979ca3c48a 2024-11-14T00:29:07,221 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/c526db6f9e5b4ad0be935f979ca3c48a, entries=150, sequenceid=159, filesize=11.9 K 2024-11-14T00:29:07,224 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for a0f5137c718f1ce8c674c5a78d56c3d3 in 1165ms, sequenceid=159, compaction requested=true 2024-11-14T00:29:07,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:07,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:07,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-14T00:29:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-14T00:29:07,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-14T00:29:07,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3510 sec 2024-11-14T00:29:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.3590 sec 2024-11-14T00:29:07,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:07,490 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:07,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T00:29:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a48a05981d02412eb958c808970b47d4_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_0/A:col10/1731544147428/Put/seqid=0 2024-11-14T00:29:07,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742174_1350 (size=17284) 2024-11-14T00:29:07,592 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:07,617 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a48a05981d02412eb958c808970b47d4_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a48a05981d02412eb958c808970b47d4_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:07,618 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/833ac59fe21442f7ba055964517c4830, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:07,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/833ac59fe21442f7ba055964517c4830 is 175, key is test_row_0/A:col10/1731544147428/Put/seqid=0 2024-11-14T00:29:07,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742175_1351 (size=48389) 2024-11-14T00:29:07,655 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/833ac59fe21442f7ba055964517c4830 2024-11-14T00:29:07,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/6f7dc9bdef59440b882c51f9800fc4c5 is 50, key is test_row_0/B:col10/1731544147428/Put/seqid=0 2024-11-14T00:29:07,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742176_1352 (size=12151) 2024-11-14T00:29:07,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:07,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544207734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:07,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544207738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:07,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544207739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:07,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:07,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544207865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:07,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:07,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544207866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:07,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:07,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:07,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:07,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:07,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:07,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544207866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:07,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:07,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:07,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:08,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T00:29:08,029 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:08,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:29:08,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-14T00:29:08,032 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:29:08,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T00:29:08,033 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:29:08,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:29:08,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:08,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544208085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:08,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544208085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:08,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:08,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544208086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T00:29:08,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/6f7dc9bdef59440b882c51f9800fc4c5 2024-11-14T00:29:08,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/95bb4438c6ea4ddf95ec2bb4541e17e0 is 50, key is test_row_0/C:col10/1731544147428/Put/seqid=0 2024-11-14T00:29:08,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T00:29:08,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:08,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:08,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:08,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742177_1353 (size=12151) 2024-11-14T00:29:08,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/95bb4438c6ea4ddf95ec2bb4541e17e0 2024-11-14T00:29:08,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/833ac59fe21442f7ba055964517c4830 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/833ac59fe21442f7ba055964517c4830 2024-11-14T00:29:08,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/833ac59fe21442f7ba055964517c4830, entries=250, sequenceid=170, filesize=47.3 K 2024-11-14T00:29:08,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/6f7dc9bdef59440b882c51f9800fc4c5 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/6f7dc9bdef59440b882c51f9800fc4c5 2024-11-14T00:29:08,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/6f7dc9bdef59440b882c51f9800fc4c5, entries=150, sequenceid=170, filesize=11.9 K 2024-11-14T00:29:08,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/95bb4438c6ea4ddf95ec2bb4541e17e0 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/95bb4438c6ea4ddf95ec2bb4541e17e0 2024-11-14T00:29:08,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/95bb4438c6ea4ddf95ec2bb4541e17e0, entries=150, sequenceid=170, filesize=11.9 K 2024-11-14T00:29:08,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for a0f5137c718f1ce8c674c5a78d56c3d3 in 785ms, sequenceid=170, compaction requested=true 2024-11-14T00:29:08,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:08,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:08,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:08,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:08,276 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:29:08,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:08,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:08,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:29:08,276 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:29:08,277 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:29:08,277 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/B is initiating minor compaction (all files) 2024-11-14T00:29:08,278 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/B in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:08,278 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/20f3a0b2734b45c4bd04492bd0496159, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/3579ab65e5db427ea88d16f6f8b4bdd2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/6df26c50bc9b45a686534f80204b38ca, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/6f7dc9bdef59440b882c51f9800fc4c5] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=47.4 K 2024-11-14T00:29:08,278 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 20f3a0b2734b45c4bd04492bd0496159, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731544142032 2024-11-14T00:29:08,278 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:29:08,279 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/A is initiating minor compaction (all files) 2024-11-14T00:29:08,279 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/A in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:08,279 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/9b13fe17f74d4cbf88274c015f7091fd, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/cd6b2eb2d6e0451fb286f15202effb04, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/8fc5275223eb4688af454ed20d47ed55, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/833ac59fe21442f7ba055964517c4830] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=138.3 K 2024-11-14T00:29:08,279 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:08,279 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/9b13fe17f74d4cbf88274c015f7091fd, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/cd6b2eb2d6e0451fb286f15202effb04, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/8fc5275223eb4688af454ed20d47ed55, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/833ac59fe21442f7ba055964517c4830] 2024-11-14T00:29:08,279 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 3579ab65e5db427ea88d16f6f8b4bdd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731544143235 2024-11-14T00:29:08,279 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b13fe17f74d4cbf88274c015f7091fd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731544142032 2024-11-14T00:29:08,280 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 6df26c50bc9b45a686534f80204b38ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731544143989 2024-11-14T00:29:08,280 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd6b2eb2d6e0451fb286f15202effb04, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731544143235 2024-11-14T00:29:08,280 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fc5275223eb4688af454ed20d47ed55, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731544143989 2024-11-14T00:29:08,280 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f7dc9bdef59440b882c51f9800fc4c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731544147428 2024-11-14T00:29:08,281 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 833ac59fe21442f7ba055964517c4830, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731544146213 2024-11-14T00:29:08,284 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:08,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-14T00:29:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:08,304 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#B#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:08,309 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/fc89dca19e03440bb8c19b01a7c00f76 is 50, key is test_row_0/B:col10/1731544147428/Put/seqid=0 2024-11-14T00:29:08,342 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T00:29:08,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b9ef57611f0647bb9c1a0739c4114c7f_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_0/A:col10/1731544147726/Put/seqid=0 2024-11-14T00:29:08,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T00:29:08,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:08,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:08,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:08,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,356 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114709cc9a259b8422f8f2e9481649d8a9e_a0f5137c718f1ce8c674c5a78d56c3d3 store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:08,359 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114709cc9a259b8422f8f2e9481649d8a9e_a0f5137c718f1ce8c674c5a78d56c3d3, store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:08,359 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114709cc9a259b8422f8f2e9481649d8a9e_a0f5137c718f1ce8c674c5a78d56c3d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:08,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544208353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:08,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:08,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544208355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:08,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:08,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:08,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742178_1354 (size=12527) 2024-11-14T00:29:08,401 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/fc89dca19e03440bb8c19b01a7c00f76 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/fc89dca19e03440bb8c19b01a7c00f76 2024-11-14T00:29:08,407 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/B of a0f5137c718f1ce8c674c5a78d56c3d3 into fc89dca19e03440bb8c19b01a7c00f76(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:08,407 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:08,408 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/B, priority=12, startTime=1731544148276; duration=0sec 2024-11-14T00:29:08,408 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:08,408 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:B 2024-11-14T00:29:08,408 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:29:08,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544208396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:08,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544208396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:08,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:08,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544208396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:08,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:08,423 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:29:08,424 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/C is initiating minor compaction (all files) 2024-11-14T00:29:08,424 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/C in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:08,424 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a839975e18244716b299abd5cc3ca768, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/cc5b90f33bdd4e78af11adef0c5266e7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/c526db6f9e5b4ad0be935f979ca3c48a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/95bb4438c6ea4ddf95ec2bb4541e17e0] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=47.4 K 2024-11-14T00:29:08,432 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting a839975e18244716b299abd5cc3ca768, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731544142032 2024-11-14T00:29:08,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742180_1356 (size=4469) 2024-11-14T00:29:08,433 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting cc5b90f33bdd4e78af11adef0c5266e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731544143235 2024-11-14T00:29:08,433 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c526db6f9e5b4ad0be935f979ca3c48a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731544143989 2024-11-14T00:29:08,434 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 95bb4438c6ea4ddf95ec2bb4541e17e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731544147428 2024-11-14T00:29:08,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742179_1355 (size=14794) 2024-11-14T00:29:08,446 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#C#compaction#300 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:08,447 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/dae1f244a9024f62ac9929b254711f39 is 50, key is test_row_0/C:col10/1731544147428/Put/seqid=0 2024-11-14T00:29:08,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742181_1357 (size=12527) 2024-11-14T00:29:08,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:08,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544208489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:08,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:08,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544208490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:08,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:08,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:08,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T00:29:08,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:08,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:08,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:08,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T00:29:08,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T00:29:08,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:08,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:08,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:08,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544208716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:08,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:08,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544208716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:08,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T00:29:08,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:08,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:08,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:08,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,836 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:08,840 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#A#compaction#298 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:08,840 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/11a10fa7e42b42d1b2210ddec758628a is 175, key is test_row_0/A:col10/1731544147428/Put/seqid=0 2024-11-14T00:29:08,848 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b9ef57611f0647bb9c1a0739c4114c7f_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b9ef57611f0647bb9c1a0739c4114c7f_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:08,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742182_1358 (size=31481) 2024-11-14T00:29:08,867 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/4882e969129e430896ecc411b6773052, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:08,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/4882e969129e430896ecc411b6773052 is 175, key is test_row_0/A:col10/1731544147726/Put/seqid=0 2024-11-14T00:29:08,879 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/11a10fa7e42b42d1b2210ddec758628a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/11a10fa7e42b42d1b2210ddec758628a 2024-11-14T00:29:08,896 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/A of a0f5137c718f1ce8c674c5a78d56c3d3 into 11a10fa7e42b42d1b2210ddec758628a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:08,897 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:08,897 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/A, priority=12, startTime=1731544148276; duration=0sec 2024-11-14T00:29:08,897 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:08,897 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:A 2024-11-14T00:29:08,913 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/dae1f244a9024f62ac9929b254711f39 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/dae1f244a9024f62ac9929b254711f39 2024-11-14T00:29:08,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742183_1359 (size=39749) 2024-11-14T00:29:08,914 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/4882e969129e430896ecc411b6773052 2024-11-14T00:29:08,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544208915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:08,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544208915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:08,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544208915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:08,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:08,923 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/C of a0f5137c718f1ce8c674c5a78d56c3d3 into dae1f244a9024f62ac9929b254711f39(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:08,923 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:08,923 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/C, priority=12, startTime=1731544148276; duration=0sec 2024-11-14T00:29:08,923 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:08,923 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:C 2024-11-14T00:29:08,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/19f67127540f42e69e8156df7e686fe7 is 50, key is test_row_0/B:col10/1731544147726/Put/seqid=0 2024-11-14T00:29:08,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T00:29:08,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:08,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:08,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:08,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:08,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742184_1360 (size=12151) 2024-11-14T00:29:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:09,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544209027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:09,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544209037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:09,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:09,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:09,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:09,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T00:29:09,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:09,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:09,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:09,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:09,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T00:29:09,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T00:29:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:09,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:09,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/19f67127540f42e69e8156df7e686fe7 2024-11-14T00:29:09,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/a3f854d5c4f944129c5c8ab3aeb7228e is 50, key is test_row_0/C:col10/1731544147726/Put/seqid=0 2024-11-14T00:29:09,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742185_1361 (size=12151) 2024-11-14T00:29:09,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T00:29:09,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:09,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:09,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:09,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:09,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:09,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544209555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:09,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:09,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:09,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:09,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544209566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:09,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:09,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:09,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:09,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T00:29:09,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:09,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:09,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:09,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:09,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:09,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T00:29:09,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:09,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:09,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:09,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:09,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:09,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/a3f854d5c4f944129c5c8ab3aeb7228e 2024-11-14T00:29:09,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/4882e969129e430896ecc411b6773052 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/4882e969129e430896ecc411b6773052 2024-11-14T00:29:09,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/4882e969129e430896ecc411b6773052, entries=200, sequenceid=197, filesize=38.8 K 2024-11-14T00:29:09,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/19f67127540f42e69e8156df7e686fe7 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/19f67127540f42e69e8156df7e686fe7 2024-11-14T00:29:09,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/19f67127540f42e69e8156df7e686fe7, entries=150, sequenceid=197, filesize=11.9 K 2024-11-14T00:29:09,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/a3f854d5c4f944129c5c8ab3aeb7228e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a3f854d5c4f944129c5c8ab3aeb7228e 2024-11-14T00:29:09,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a3f854d5c4f944129c5c8ab3aeb7228e, entries=150, sequenceid=197, filesize=11.9 K 2024-11-14T00:29:09,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for a0f5137c718f1ce8c674c5a78d56c3d3 in 1605ms, sequenceid=197, compaction requested=false 2024-11-14T00:29:09,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:09,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T00:29:09,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:09,941 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T00:29:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:09,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:09,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cf525681f7d648aface80c3060976ea0_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_0/A:col10/1731544148330/Put/seqid=0 2024-11-14T00:29:10,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742186_1362 (size=12304) 2024-11-14T00:29:10,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:10,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544210081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:10,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544210092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:10,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544210098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:10,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T00:29:10,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544210208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:10,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:10,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544210215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:10,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544210220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:10,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:10,422 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cf525681f7d648aface80c3060976ea0_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cf525681f7d648aface80c3060976ea0_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/6db57e3f36f049d39867907aa10c1bb2, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/6db57e3f36f049d39867907aa10c1bb2 is 175, key is test_row_0/A:col10/1731544148330/Put/seqid=0 2024-11-14T00:29:10,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544210428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:10,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544210437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:10,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742187_1363 (size=31105) 2024-11-14T00:29:10,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544210440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:10,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:10,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544210580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:10,590 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_2' in a0f5137c718f1ce8c674c5a78d56c3d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:10,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544210588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:10,604 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_1' in a0f5137c718f1ce8c674c5a78d56c3d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:10,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544210755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:10,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:10,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544210762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:10,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544210765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:10,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:10,866 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/6db57e3f36f049d39867907aa10c1bb2 2024-11-14T00:29:10,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/76d5fb431bcc4798bb548aaf0934bb36 is 50, key is test_row_0/B:col10/1731544148330/Put/seqid=0 2024-11-14T00:29:10,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742188_1364 (size=12151) 2024-11-14T00:29:11,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:11,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544211275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:11,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:11,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:11,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:11,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:11,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544211285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:11,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544211285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:11,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:11,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:11,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:11,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:11,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:11,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:11,362 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/76d5fb431bcc4798bb548aaf0934bb36 2024-11-14T00:29:11,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/d3b08719d29944948ffb8390bdaa23cf is 50, key is test_row_0/C:col10/1731544148330/Put/seqid=0 2024-11-14T00:29:11,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742189_1365 (size=12151) 2024-11-14T00:29:11,398 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/d3b08719d29944948ffb8390bdaa23cf 2024-11-14T00:29:11,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/6db57e3f36f049d39867907aa10c1bb2 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/6db57e3f36f049d39867907aa10c1bb2 2024-11-14T00:29:11,410 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/6db57e3f36f049d39867907aa10c1bb2, entries=150, sequenceid=210, filesize=30.4 K 2024-11-14T00:29:11,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/76d5fb431bcc4798bb548aaf0934bb36 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/76d5fb431bcc4798bb548aaf0934bb36 2024-11-14T00:29:11,431 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/76d5fb431bcc4798bb548aaf0934bb36, entries=150, sequenceid=210, filesize=11.9 K 2024-11-14T00:29:11,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/d3b08719d29944948ffb8390bdaa23cf as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/d3b08719d29944948ffb8390bdaa23cf 2024-11-14T00:29:11,441 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/d3b08719d29944948ffb8390bdaa23cf, entries=150, sequenceid=210, filesize=11.9 K 2024-11-14T00:29:11,442 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for a0f5137c718f1ce8c674c5a78d56c3d3 in 1501ms, sequenceid=210, compaction requested=true 2024-11-14T00:29:11,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:11,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:11,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-14T00:29:11,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-14T00:29:11,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-14T00:29:11,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4110 sec 2024-11-14T00:29:11,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 3.4170 sec 2024-11-14T00:29:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T00:29:12,196 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:12,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:29:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-14T00:29:12,212 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:29:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T00:29:12,224 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:29:12,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:29:12,318 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:12,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-14T00:29:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T00:29:12,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114df1d79d46c594b5b88de0acd9b9f285c_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_0/A:col10/1731544150096/Put/seqid=0 2024-11-14T00:29:12,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544212362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:12,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742190_1366 (size=14794) 2024-11-14T00:29:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:12,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544212375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:12,402 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:12,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T00:29:12,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:12,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:12,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:12,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:12,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:12,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544212377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:12,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:12,429 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114df1d79d46c594b5b88de0acd9b9f285c_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114df1d79d46c594b5b88de0acd9b9f285c_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:12,441 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/00d91f9fd7164577bbc8140db252c5bf, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:12,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/00d91f9fd7164577bbc8140db252c5bf is 175, key is test_row_0/A:col10/1731544150096/Put/seqid=0 2024-11-14T00:29:12,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742191_1367 (size=39749) 2024-11-14T00:29:12,486 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/00d91f9fd7164577bbc8140db252c5bf 2024-11-14T00:29:12,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544212505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:12,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544212505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:12,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/1abd0bd89f0b477699e9f17c3b2cbe19 is 50, key is test_row_0/B:col10/1731544150096/Put/seqid=0 2024-11-14T00:29:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T00:29:12,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544212535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:12,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:12,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T00:29:12,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:12,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:12,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742192_1368 (size=12151) 2024-11-14T00:29:12,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/1abd0bd89f0b477699e9f17c3b2cbe19 2024-11-14T00:29:12,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/494f8e9cdfb74b0cbe61e68c441be91b is 50, key is test_row_0/C:col10/1731544150096/Put/seqid=0 2024-11-14T00:29:12,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544212609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:12,634 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_2' in a0f5137c718f1ce8c674c5a78d56c3d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:12,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544212628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:12,637 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_1' in a0f5137c718f1ce8c674c5a78d56c3d3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4283 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:12,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742193_1369 (size=12151) 2024-11-14T00:29:12,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T00:29:12,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:12,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:12,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:12,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:12,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:12,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544212726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544212728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:12,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544212767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:12,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T00:29:12,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T00:29:12,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:12,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:12,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:12,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:12,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:12,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:13,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T00:29:13,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:13,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:13,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:13,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:13,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:13,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:13,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:13,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544213055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:13,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544213056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:13,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/494f8e9cdfb74b0cbe61e68c441be91b 2024-11-14T00:29:13,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/00d91f9fd7164577bbc8140db252c5bf as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/00d91f9fd7164577bbc8140db252c5bf 2024-11-14T00:29:13,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/00d91f9fd7164577bbc8140db252c5bf, entries=200, sequenceid=237, filesize=38.8 K 2024-11-14T00:29:13,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/1abd0bd89f0b477699e9f17c3b2cbe19 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/1abd0bd89f0b477699e9f17c3b2cbe19 2024-11-14T00:29:13,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:13,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544213089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:13,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:13,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/1abd0bd89f0b477699e9f17c3b2cbe19, entries=150, sequenceid=237, filesize=11.9 K 2024-11-14T00:29:13,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/494f8e9cdfb74b0cbe61e68c441be91b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/494f8e9cdfb74b0cbe61e68c441be91b 2024-11-14T00:29:13,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/494f8e9cdfb74b0cbe61e68c441be91b, entries=150, sequenceid=237, filesize=11.9 K 2024-11-14T00:29:13,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for a0f5137c718f1ce8c674c5a78d56c3d3 in 817ms, sequenceid=237, compaction requested=true 2024-11-14T00:29:13,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:13,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:13,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:13,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:13,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:29:13,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:13,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T00:29:13,137 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:29:13,138 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:29:13,141 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142084 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:29:13,142 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/A is initiating minor compaction (all files) 2024-11-14T00:29:13,142 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/A in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:13,142 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/11a10fa7e42b42d1b2210ddec758628a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/4882e969129e430896ecc411b6773052, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/6db57e3f36f049d39867907aa10c1bb2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/00d91f9fd7164577bbc8140db252c5bf] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=138.8 K 2024-11-14T00:29:13,142 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:13,142 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/11a10fa7e42b42d1b2210ddec758628a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/4882e969129e430896ecc411b6773052, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/6db57e3f36f049d39867907aa10c1bb2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/00d91f9fd7164577bbc8140db252c5bf] 2024-11-14T00:29:13,144 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11a10fa7e42b42d1b2210ddec758628a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731544147428 2024-11-14T00:29:13,146 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4882e969129e430896ecc411b6773052, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731544147707 2024-11-14T00:29:13,147 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6db57e3f36f049d39867907aa10c1bb2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731544148329 2024-11-14T00:29:13,148 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00d91f9fd7164577bbc8140db252c5bf, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731544150075 2024-11-14T00:29:13,149 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:29:13,149 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/C is initiating minor compaction (all files) 2024-11-14T00:29:13,149 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/C in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:13,149 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/dae1f244a9024f62ac9929b254711f39, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a3f854d5c4f944129c5c8ab3aeb7228e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/d3b08719d29944948ffb8390bdaa23cf, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/494f8e9cdfb74b0cbe61e68c441be91b] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=47.8 K 2024-11-14T00:29:13,150 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting dae1f244a9024f62ac9929b254711f39, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731544147428 2024-11-14T00:29:13,150 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting a3f854d5c4f944129c5c8ab3aeb7228e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731544147707 2024-11-14T00:29:13,157 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting d3b08719d29944948ffb8390bdaa23cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731544148329 2024-11-14T00:29:13,161 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 494f8e9cdfb74b0cbe61e68c441be91b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731544150087 2024-11-14T00:29:13,182 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:13,188 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#C#compaction#310 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:13,189 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/5017a2ca4f164d03b81691cd02c19df0 is 50, key is test_row_0/C:col10/1731544150096/Put/seqid=0 2024-11-14T00:29:13,202 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114f464f9424d134828a75991a5acf6caab_a0f5137c718f1ce8c674c5a78d56c3d3 store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:13,205 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114f464f9424d134828a75991a5acf6caab_a0f5137c718f1ce8c674c5a78d56c3d3, store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:13,205 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f464f9424d134828a75991a5acf6caab_a0f5137c718f1ce8c674c5a78d56c3d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:13,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T00:29:13,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:13,206 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-14T00:29:13,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742195_1371 (size=4469) 2024-11-14T00:29:13,237 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#A#compaction#309 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:13,237 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/a79fc430ec9e41e4a85583b773c8ae28 is 175, key is test_row_0/A:col10/1731544150096/Put/seqid=0 2024-11-14T00:29:13,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111495d741eb29b44f5cb2ef2d1827d2cb2f_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_0/A:col10/1731544152355/Put/seqid=0 2024-11-14T00:29:13,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742194_1370 (size=12663) 2024-11-14T00:29:13,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742196_1372 (size=31617) 2024-11-14T00:29:13,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742197_1373 (size=12304) 2024-11-14T00:29:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T00:29:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:13,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:13,708 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/a79fc430ec9e41e4a85583b773c8ae28 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/a79fc430ec9e41e4a85583b773c8ae28 2024-11-14T00:29:13,721 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/5017a2ca4f164d03b81691cd02c19df0 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/5017a2ca4f164d03b81691cd02c19df0 2024-11-14T00:29:13,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:13,726 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/A of a0f5137c718f1ce8c674c5a78d56c3d3 into a79fc430ec9e41e4a85583b773c8ae28(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:13,727 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:13,727 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/A, priority=12, startTime=1731544153135; duration=0sec 2024-11-14T00:29:13,731 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/C of a0f5137c718f1ce8c674c5a78d56c3d3 into 5017a2ca4f164d03b81691cd02c19df0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:13,731 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:13,731 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/C, priority=12, startTime=1731544153135; duration=0sec 2024-11-14T00:29:13,731 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:13,731 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:C 2024-11-14T00:29:13,731 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:29:13,727 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:13,732 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:A 2024-11-14T00:29:13,735 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111495d741eb29b44f5cb2ef2d1827d2cb2f_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111495d741eb29b44f5cb2ef2d1827d2cb2f_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/bcfc2b26e3f64f48bcdd9dc76bd25970, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:13,737 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:29:13,738 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/B is initiating minor compaction (all files) 2024-11-14T00:29:13,738 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/B in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:13,738 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/fc89dca19e03440bb8c19b01a7c00f76, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/19f67127540f42e69e8156df7e686fe7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/76d5fb431bcc4798bb548aaf0934bb36, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/1abd0bd89f0b477699e9f17c3b2cbe19] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=47.8 K 2024-11-14T00:29:13,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/bcfc2b26e3f64f48bcdd9dc76bd25970 is 175, key is test_row_0/A:col10/1731544152355/Put/seqid=0 2024-11-14T00:29:13,738 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting fc89dca19e03440bb8c19b01a7c00f76, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731544147428 2024-11-14T00:29:13,738 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 19f67127540f42e69e8156df7e686fe7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731544147707 2024-11-14T00:29:13,739 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 76d5fb431bcc4798bb548aaf0934bb36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731544148329 2024-11-14T00:29:13,740 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 1abd0bd89f0b477699e9f17c3b2cbe19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731544150087 2024-11-14T00:29:13,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742198_1374 (size=31105) 2024-11-14T00:29:13,755 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#B#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:13,755 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/514910f5300a44c7b78ca14690158280 is 50, key is test_row_0/B:col10/1731544150096/Put/seqid=0 2024-11-14T00:29:13,757 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/bcfc2b26e3f64f48bcdd9dc76bd25970 2024-11-14T00:29:13,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544213749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:13,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544213751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:13,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544213752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:13,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742199_1375 (size=12663) 2024-11-14T00:29:13,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/84a188d22c3f4c4c8b3ec79e9f1f725a is 50, key is test_row_0/B:col10/1731544152355/Put/seqid=0 2024-11-14T00:29:13,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742200_1376 (size=12151) 2024-11-14T00:29:13,843 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/84a188d22c3f4c4c8b3ec79e9f1f725a 2024-11-14T00:29:13,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/b1d207c8cf144348ba41f66d0bffb676 is 50, key is test_row_0/C:col10/1731544152355/Put/seqid=0 2024-11-14T00:29:13,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:13,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544213866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:13,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544213868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:13,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544213875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:13,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:13,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742201_1377 (size=12151) 2024-11-14T00:29:14,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544214089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:14,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544214090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:14,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:14,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544214096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:14,204 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/514910f5300a44c7b78ca14690158280 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/514910f5300a44c7b78ca14690158280 2024-11-14T00:29:14,211 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/B of a0f5137c718f1ce8c674c5a78d56c3d3 into 514910f5300a44c7b78ca14690158280(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:14,211 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:14,211 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/B, priority=12, startTime=1731544153135; duration=0sec 2024-11-14T00:29:14,211 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:14,211 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:B 2024-11-14T00:29:14,304 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/b1d207c8cf144348ba41f66d0bffb676 2024-11-14T00:29:14,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/bcfc2b26e3f64f48bcdd9dc76bd25970 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bcfc2b26e3f64f48bcdd9dc76bd25970 2024-11-14T00:29:14,333 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bcfc2b26e3f64f48bcdd9dc76bd25970, entries=150, sequenceid=246, filesize=30.4 K 2024-11-14T00:29:14,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/84a188d22c3f4c4c8b3ec79e9f1f725a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/84a188d22c3f4c4c8b3ec79e9f1f725a 2024-11-14T00:29:14,340 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/84a188d22c3f4c4c8b3ec79e9f1f725a, entries=150, sequenceid=246, filesize=11.9 K 2024-11-14T00:29:14,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/b1d207c8cf144348ba41f66d0bffb676 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/b1d207c8cf144348ba41f66d0bffb676 2024-11-14T00:29:14,348 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/b1d207c8cf144348ba41f66d0bffb676, entries=150, sequenceid=246, filesize=11.9 K 2024-11-14T00:29:14,354 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for a0f5137c718f1ce8c674c5a78d56c3d3 in 1148ms, sequenceid=246, compaction requested=false 2024-11-14T00:29:14,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:14,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:14,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-14T00:29:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T00:29:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-14T00:29:14,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-14T00:29:14,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1340 sec 2024-11-14T00:29:14,361 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.1550 sec 2024-11-14T00:29:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:14,412 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:14,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-14T00:29:14,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111444dae1f1c19c4041804eb256c44f38c7_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_0/A:col10/1731544153723/Put/seqid=0 2024-11-14T00:29:14,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544214433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:14,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:14,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742202_1378 (size=17534) 2024-11-14T00:29:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544214434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:14,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:14,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544214437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:14,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:14,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544214545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:14,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544214555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:14,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544214556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:14,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544214756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:14,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544214765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544214765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:14,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:14,845 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:14,852 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111444dae1f1c19c4041804eb256c44f38c7_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111444dae1f1c19c4041804eb256c44f38c7_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:14,853 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/bab549e0976944548176d6b16bc171fc, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:14,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/bab549e0976944548176d6b16bc171fc is 175, key is test_row_0/A:col10/1731544153723/Put/seqid=0 2024-11-14T00:29:14,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742203_1379 (size=48639) 2024-11-14T00:29:15,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544215080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:15,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544215088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:15,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:15,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:15,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:15,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:15,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:15,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544215100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:15,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:15,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:15,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:15,311 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/bab549e0976944548176d6b16bc171fc 2024-11-14T00:29:15,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/5ec0d66a830b460fb40f8334e1d565d2 is 50, key is test_row_0/B:col10/1731544153723/Put/seqid=0 2024-11-14T00:29:15,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742204_1380 (size=12301) 2024-11-14T00:29:15,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/5ec0d66a830b460fb40f8334e1d565d2 2024-11-14T00:29:15,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/a551e386985442e88d69d94339133c24 is 50, key is test_row_0/C:col10/1731544153723/Put/seqid=0 2024-11-14T00:29:15,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742205_1381 (size=12301) 2024-11-14T00:29:15,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544215595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:15,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:15,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544215605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:15,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:15,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:15,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544215615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:15,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/a551e386985442e88d69d94339133c24 2024-11-14T00:29:15,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/bab549e0976944548176d6b16bc171fc as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bab549e0976944548176d6b16bc171fc 2024-11-14T00:29:15,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bab549e0976944548176d6b16bc171fc, entries=250, sequenceid=278, filesize=47.5 K 2024-11-14T00:29:15,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/5ec0d66a830b460fb40f8334e1d565d2 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/5ec0d66a830b460fb40f8334e1d565d2 2024-11-14T00:29:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/5ec0d66a830b460fb40f8334e1d565d2, entries=150, sequenceid=278, filesize=12.0 K 2024-11-14T00:29:15,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/a551e386985442e88d69d94339133c24 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a551e386985442e88d69d94339133c24 2024-11-14T00:29:15,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a551e386985442e88d69d94339133c24, entries=150, sequenceid=278, filesize=12.0 K 2024-11-14T00:29:15,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for a0f5137c718f1ce8c674c5a78d56c3d3 in 1436ms, sequenceid=278, compaction requested=true 2024-11-14T00:29:15,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:15,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:15,849 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:15,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:15,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:15,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:15,849 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:15,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:15,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:15,850 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:15,850 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/B is initiating minor compaction (all files) 2024-11-14T00:29:15,850 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/B in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:15,850 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/514910f5300a44c7b78ca14690158280, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/84a188d22c3f4c4c8b3ec79e9f1f725a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/5ec0d66a830b460fb40f8334e1d565d2] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=36.2 K 2024-11-14T00:29:15,851 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111361 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:15,851 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/A is initiating minor compaction (all files) 2024-11-14T00:29:15,851 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/A in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:15,851 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/a79fc430ec9e41e4a85583b773c8ae28, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bcfc2b26e3f64f48bcdd9dc76bd25970, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bab549e0976944548176d6b16bc171fc] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=108.8 K 2024-11-14T00:29:15,851 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:15,851 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/a79fc430ec9e41e4a85583b773c8ae28, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bcfc2b26e3f64f48bcdd9dc76bd25970, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bab549e0976944548176d6b16bc171fc] 2024-11-14T00:29:15,851 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 514910f5300a44c7b78ca14690158280, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731544150087 2024-11-14T00:29:15,852 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting a79fc430ec9e41e4a85583b773c8ae28, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731544150087 2024-11-14T00:29:15,852 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 84a188d22c3f4c4c8b3ec79e9f1f725a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731544152352 2024-11-14T00:29:15,852 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcfc2b26e3f64f48bcdd9dc76bd25970, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731544152352 2024-11-14T00:29:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,853 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ec0d66a830b460fb40f8334e1d565d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731544153723 2024-11-14T00:29:15,853 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting bab549e0976944548176d6b16bc171fc, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731544153723 2024-11-14T00:29:15,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,873 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,887 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#B#compaction#319 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,888 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/c51ba0fee553455ea2b056b4bf7b7ab6 is 50, key is test_row_0/B:col10/1731544153723/Put/seqid=0 2024-11-14T00:29:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,896 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411146a49aee3ccd84c72b40957c06a6b7da4_a0f5137c718f1ce8c674c5a78d56c3d3 store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,899 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411146a49aee3ccd84c72b40957c06a6b7da4_a0f5137c718f1ce8c674c5a78d56c3d3, store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:15,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,899 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146a49aee3ccd84c72b40957c06a6b7da4_a0f5137c718f1ce8c674c5a78d56c3d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:15,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742206_1382 (size=12915) 2024-11-14T00:29:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742207_1383 (size=4469) 2024-11-14T00:29:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,361 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/c51ba0fee553455ea2b056b4bf7b7ab6 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/c51ba0fee553455ea2b056b4bf7b7ab6 2024-11-14T00:29:16,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T00:29:16,365 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:29:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-14T00:29:16,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,369 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/B of a0f5137c718f1ce8c674c5a78d56c3d3 into c51ba0fee553455ea2b056b4bf7b7ab6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:16,369 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:16,369 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/B, priority=13, startTime=1731544155849; duration=0sec 2024-11-14T00:29:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T00:29:16,369 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:16,369 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:B 2024-11-14T00:29:16,369 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:29:16,369 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:16,370 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#A#compaction#318 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,370 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/50878588dc954ee99f88f95b06723062 is 175, key is test_row_0/A:col10/1731544153723/Put/seqid=0 2024-11-14T00:29:16,370 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:29:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:29:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,372 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:16,373 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/C is initiating minor compaction (all files) 2024-11-14T00:29:16,373 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/C in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:16,373 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/5017a2ca4f164d03b81691cd02c19df0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/b1d207c8cf144348ba41f66d0bffb676, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a551e386985442e88d69d94339133c24] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=36.2 K 2024-11-14T00:29:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,373 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 5017a2ca4f164d03b81691cd02c19df0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731544150087 2024-11-14T00:29:16,374 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting b1d207c8cf144348ba41f66d0bffb676, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731544152352 2024-11-14T00:29:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,374 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting a551e386985442e88d69d94339133c24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731544153723 2024-11-14T00:29:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,403 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#C#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,403 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/0aa5576f57054024b8479711b7aa67eb is 50, key is test_row_0/C:col10/1731544153723/Put/seqid=0 2024-11-14T00:29:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742208_1384 (size=31869) 2024-11-14T00:29:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,417 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/50878588dc954ee99f88f95b06723062 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/50878588dc954ee99f88f95b06723062 2024-11-14T00:29:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742209_1385 (size=12915) 2024-11-14T00:29:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,424 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/A of a0f5137c718f1ce8c674c5a78d56c3d3 into 50878588dc954ee99f88f95b06723062(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:16,424 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:16,424 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/A, priority=13, startTime=1731544155849; duration=0sec 2024-11-14T00:29:16,425 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:16,425 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:A 2024-11-14T00:29:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,425 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/0aa5576f57054024b8479711b7aa67eb as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/0aa5576f57054024b8479711b7aa67eb 2024-11-14T00:29:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,441 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/C of a0f5137c718f1ce8c674c5a78d56c3d3 into 0aa5576f57054024b8479711b7aa67eb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:16,441 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:16,441 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/C, priority=13, startTime=1731544155849; duration=0sec 2024-11-14T00:29:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,441 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:16,441 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:C 2024-11-14T00:29:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T00:29:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T00:29:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:16,533 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-14T00:29:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114dbbe89f38ec340a5957ee96fb1e9f3e1_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_1/A:col10/1731544154434/Put/seqid=0 2024-11-14T00:29:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742210_1386 (size=9914) 2024-11-14T00:29:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,587 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114dbbe89f38ec340a5957ee96fb1e9f3e1_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dbbe89f38ec340a5957ee96fb1e9f3e1_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/860744401ba94269aab4b9d8bb6a19cc, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/860744401ba94269aab4b9d8bb6a19cc is 175, key is test_row_1/A:col10/1731544154434/Put/seqid=0 2024-11-14T00:29:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742211_1387 (size=22561) 2024-11-14T00:29:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,594 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/860744401ba94269aab4b9d8bb6a19cc 2024-11-14T00:29:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/8304806389a0475d803289ee8f931cf2 is 50, key is test_row_1/B:col10/1731544154434/Put/seqid=0 2024-11-14T00:29:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742212_1388 (size=9857) 2024-11-14T00:29:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. as already flushing 2024-11-14T00:29:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T00:29:16,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:16,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544216730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:16,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:16,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544216739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:16,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:16,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544216742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:16,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:16,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544216742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:16,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:16,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544216743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:16,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544216845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:16,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544216857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:16,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:16,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544216865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:16,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544216866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544216866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:16,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:16,967 INFO [Thread-1439 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1439. 2024-11-14T00:29:16,967 DEBUG [Thread-1439 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:16,967 DEBUG [Thread-1439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:16,967 DEBUG [Thread-1439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:16,967 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:16,968 INFO [Thread-1437 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1437. 2024-11-14T00:29:16,968 DEBUG [Thread-1437 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:16,968 DEBUG [Thread-1437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:16,968 DEBUG [Thread-1437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:16,968 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:16,972 INFO [Thread-1441 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1441. 2024-11-14T00:29:16,972 DEBUG [Thread-1441 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:16,972 DEBUG [Thread-1441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:16,972 DEBUG [Thread-1441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:16,972 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:16,978 INFO [Thread-1435 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1435. 2024-11-14T00:29:16,978 DEBUG [Thread-1435 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:16,978 DEBUG [Thread-1435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:16,978 DEBUG [Thread-1435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:16,978 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:16,981 INFO [Thread-1433 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1433. 2024-11-14T00:29:16,981 DEBUG [Thread-1433 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:16,981 DEBUG [Thread-1433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:16,981 DEBUG [Thread-1433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:16,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T00:29:17,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:17,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544217056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:17,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:17,061 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/8304806389a0475d803289ee8f931cf2 2024-11-14T00:29:17,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/8695c3a0a57642a1ae13d323c20a9a98 is 50, key is test_row_1/C:col10/1731544154434/Put/seqid=0 2024-11-14T00:29:17,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544217075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:17,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742213_1389 (size=9857) 2024-11-14T00:29:17,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:17,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:17,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544217085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544217085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:17,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544217085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:17,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51456 deadline: 1731544217365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:17,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51454 deadline: 1731544217385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:17,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:17,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:17,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51482 deadline: 1731544217396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544217396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51470 deadline: 1731544217396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:17,482 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/8695c3a0a57642a1ae13d323c20a9a98 2024-11-14T00:29:17,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/860744401ba94269aab4b9d8bb6a19cc as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/860744401ba94269aab4b9d8bb6a19cc 2024-11-14T00:29:17,493 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/860744401ba94269aab4b9d8bb6a19cc, entries=100, sequenceid=288, filesize=22.0 K 2024-11-14T00:29:17,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/8304806389a0475d803289ee8f931cf2 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/8304806389a0475d803289ee8f931cf2 2024-11-14T00:29:17,499 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/8304806389a0475d803289ee8f931cf2, entries=100, sequenceid=288, filesize=9.6 K 2024-11-14T00:29:17,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/8695c3a0a57642a1ae13d323c20a9a98 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/8695c3a0a57642a1ae13d323c20a9a98 2024-11-14T00:29:17,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T00:29:17,511 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/8695c3a0a57642a1ae13d323c20a9a98, entries=100, sequenceid=288, filesize=9.6 K 2024-11-14T00:29:17,512 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=181.14 KB/185490 for a0f5137c718f1ce8c674c5a78d56c3d3 in 980ms, sequenceid=288, compaction requested=false 2024-11-14T00:29:17,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:17,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:17,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-14T00:29:17,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-14T00:29:17,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-14T00:29:17,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1450 sec 2024-11-14T00:29:17,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.1510 sec 2024-11-14T00:29:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:17,877 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:17,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-14T00:29:17,878 INFO [Thread-1428 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1428. 2024-11-14T00:29:17,878 DEBUG [Thread-1428 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:17,878 DEBUG [Thread-1428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:17,878 DEBUG [Thread-1428 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:17,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:17,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111464a183947cc44e94a956943652e766a3_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_0/A:col10/1731544157876/Put/seqid=0 2024-11-14T00:29:17,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742214_1390 (size=12454) 2024-11-14T00:29:17,897 INFO [Thread-1426 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1426. 2024-11-14T00:29:17,897 DEBUG [Thread-1426 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:17,897 DEBUG [Thread-1426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:17,897 DEBUG [Thread-1426 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:17,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:17,910 INFO [Thread-1430 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1430. 2024-11-14T00:29:17,910 INFO [Thread-1422 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1422. 2024-11-14T00:29:17,910 DEBUG [Thread-1430 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:17,910 DEBUG [Thread-1430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:17,910 DEBUG [Thread-1422 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:17,910 DEBUG [Thread-1422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:17,910 DEBUG [Thread-1430 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:17,910 DEBUG [Thread-1422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:17,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:17,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:17,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544217909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:17,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:18,290 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:18,293 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111464a183947cc44e94a956943652e766a3_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111464a183947cc44e94a956943652e766a3_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:18,294 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/7e354f60103f4bbda8f700189afa369a, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:18,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/7e354f60103f4bbda8f700189afa369a is 175, key is test_row_0/A:col10/1731544157876/Put/seqid=0 2024-11-14T00:29:18,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742215_1391 (size=31255) 2024-11-14T00:29:18,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T00:29:18,525 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:18,698 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/7e354f60103f4bbda8f700189afa369a 2024-11-14T00:29:18,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/338565bcfbc74779a61a596be75dd50a is 50, key is test_row_0/B:col10/1731544157876/Put/seqid=0 2024-11-14T00:29:18,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742216_1392 (size=12301) 2024-11-14T00:29:18,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/338565bcfbc74779a61a596be75dd50a 2024-11-14T00:29:18,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/90f3c96b5f39401783e90d3ffd3639ac is 50, key is test_row_0/C:col10/1731544157876/Put/seqid=0 2024-11-14T00:29:18,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742217_1393 (size=12301) 2024-11-14T00:29:18,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:51480 deadline: 1731544218925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:18,926 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_0' in a0f5137c718f1ce8c674c5a78d56c3d3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:19,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/90f3c96b5f39401783e90d3ffd3639ac 2024-11-14T00:29:19,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/7e354f60103f4bbda8f700189afa369a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/7e354f60103f4bbda8f700189afa369a 2024-11-14T00:29:19,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/7e354f60103f4bbda8f700189afa369a, entries=150, sequenceid=319, filesize=30.5 K 2024-11-14T00:29:19,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/338565bcfbc74779a61a596be75dd50a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/338565bcfbc74779a61a596be75dd50a 2024-11-14T00:29:19,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/338565bcfbc74779a61a596be75dd50a, entries=150, sequenceid=319, filesize=12.0 K 2024-11-14T00:29:19,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/90f3c96b5f39401783e90d3ffd3639ac as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/90f3c96b5f39401783e90d3ffd3639ac 2024-11-14T00:29:19,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/90f3c96b5f39401783e90d3ffd3639ac, entries=150, sequenceid=319, filesize=12.0 K 2024-11-14T00:29:19,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=20.13 KB/20610 for a0f5137c718f1ce8c674c5a78d56c3d3 in 1287ms, sequenceid=319, compaction requested=true 2024-11-14T00:29:19,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:19,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:19,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:19,164 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:19,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:19,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:19,164 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:19,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0f5137c718f1ce8c674c5a78d56c3d3:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:19,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:19,166 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:19,166 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:19,166 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/B is initiating minor compaction (all files) 2024-11-14T00:29:19,166 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/A is initiating minor compaction (all files) 2024-11-14T00:29:19,166 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/B in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:19,166 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/A in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:19,166 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/c51ba0fee553455ea2b056b4bf7b7ab6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/8304806389a0475d803289ee8f931cf2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/338565bcfbc74779a61a596be75dd50a] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=34.3 K 2024-11-14T00:29:19,166 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/50878588dc954ee99f88f95b06723062, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/860744401ba94269aab4b9d8bb6a19cc, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/7e354f60103f4bbda8f700189afa369a] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=83.7 K 2024-11-14T00:29:19,166 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:19,166 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/50878588dc954ee99f88f95b06723062, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/860744401ba94269aab4b9d8bb6a19cc, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/7e354f60103f4bbda8f700189afa369a] 2024-11-14T00:29:19,166 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c51ba0fee553455ea2b056b4bf7b7ab6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731544153723 2024-11-14T00:29:19,166 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 8304806389a0475d803289ee8f931cf2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731544154431 2024-11-14T00:29:19,166 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50878588dc954ee99f88f95b06723062, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731544153723 2024-11-14T00:29:19,167 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 338565bcfbc74779a61a596be75dd50a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731544156704 2024-11-14T00:29:19,167 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 860744401ba94269aab4b9d8bb6a19cc, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731544154431 2024-11-14T00:29:19,167 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e354f60103f4bbda8f700189afa369a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731544156704 2024-11-14T00:29:19,175 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:19,176 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#B#compaction#328 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:19,177 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/2c52826e578c4afcadca3e1bfd5bfcc5 is 50, key is test_row_0/B:col10/1731544157876/Put/seqid=0 2024-11-14T00:29:19,184 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114233ff8b3bff44081b8bbaaeb9fa0a654_a0f5137c718f1ce8c674c5a78d56c3d3 store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:19,192 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114233ff8b3bff44081b8bbaaeb9fa0a654_a0f5137c718f1ce8c674c5a78d56c3d3, store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:19,192 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114233ff8b3bff44081b8bbaaeb9fa0a654_a0f5137c718f1ce8c674c5a78d56c3d3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:19,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742218_1394 (size=13017) 2024-11-14T00:29:19,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742219_1395 (size=4469) 2024-11-14T00:29:19,236 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#A#compaction#327 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:19,237 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/3732982f85c5458389d12597a83d4543 is 175, key is test_row_0/A:col10/1731544157876/Put/seqid=0 2024-11-14T00:29:19,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742220_1396 (size=31971) 2024-11-14T00:29:19,307 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/3732982f85c5458389d12597a83d4543 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/3732982f85c5458389d12597a83d4543 2024-11-14T00:29:19,313 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/A of a0f5137c718f1ce8c674c5a78d56c3d3 into 3732982f85c5458389d12597a83d4543(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:19,313 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:19,313 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/A, priority=13, startTime=1731544159164; duration=0sec 2024-11-14T00:29:19,313 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:19,313 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:A 2024-11-14T00:29:19,313 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:19,314 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:19,314 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): a0f5137c718f1ce8c674c5a78d56c3d3/C is initiating minor compaction (all files) 2024-11-14T00:29:19,314 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0f5137c718f1ce8c674c5a78d56c3d3/C in TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:19,314 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/0aa5576f57054024b8479711b7aa67eb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/8695c3a0a57642a1ae13d323c20a9a98, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/90f3c96b5f39401783e90d3ffd3639ac] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp, totalSize=34.3 K 2024-11-14T00:29:19,314 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aa5576f57054024b8479711b7aa67eb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731544153723 2024-11-14T00:29:19,315 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8695c3a0a57642a1ae13d323c20a9a98, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731544154431 2024-11-14T00:29:19,315 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90f3c96b5f39401783e90d3ffd3639ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731544156704 2024-11-14T00:29:19,323 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0f5137c718f1ce8c674c5a78d56c3d3#C#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:19,324 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/2789fde2c0ac46d89e0e84f1115ca541 is 50, key is test_row_0/C:col10/1731544157876/Put/seqid=0 2024-11-14T00:29:19,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742221_1397 (size=13017) 2024-11-14T00:29:19,609 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/2c52826e578c4afcadca3e1bfd5bfcc5 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/2c52826e578c4afcadca3e1bfd5bfcc5 2024-11-14T00:29:19,615 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/B of a0f5137c718f1ce8c674c5a78d56c3d3 into 2c52826e578c4afcadca3e1bfd5bfcc5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:19,615 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:19,615 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/B, priority=13, startTime=1731544159164; duration=0sec 2024-11-14T00:29:19,615 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:19,615 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:B 2024-11-14T00:29:19,788 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/2789fde2c0ac46d89e0e84f1115ca541 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/2789fde2c0ac46d89e0e84f1115ca541 2024-11-14T00:29:19,808 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0f5137c718f1ce8c674c5a78d56c3d3/C of a0f5137c718f1ce8c674c5a78d56c3d3 into 2789fde2c0ac46d89e0e84f1115ca541(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:19,808 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0f5137c718f1ce8c674c5a78d56c3d3: 2024-11-14T00:29:19,808 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., storeName=a0f5137c718f1ce8c674c5a78d56c3d3/C, priority=13, startTime=1731544159164; duration=0sec 2024-11-14T00:29:19,808 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:19,808 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0f5137c718f1ce8c674c5a78d56c3d3:C 2024-11-14T00:29:20,947 INFO [Thread-1424 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1424. 2024-11-14T00:29:20,947 DEBUG [Thread-1424 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:20,947 DEBUG [Thread-1424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:20,947 DEBUG [Thread-1424 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:20,947 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 912 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2736 rows 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 902 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2706 rows 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 897 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2687 rows 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 916 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2747 rows 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 905 2024-11-14T00:29:20,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2702 rows 2024-11-14T00:29:20,948 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T00:29:20,948 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T00:29:20,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:20,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:20,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:20,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-14T00:29:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T00:29:20,954 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544160954"}]},"ts":"1731544160954"} 2024-11-14T00:29:20,957 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T00:29:20,957 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T00:29:20,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T00:29:20,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0f5137c718f1ce8c674c5a78d56c3d3, UNASSIGN}] 2024-11-14T00:29:20,965 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0f5137c718f1ce8c674c5a78d56c3d3, UNASSIGN 2024-11-14T00:29:20,966 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=a0f5137c718f1ce8c674c5a78d56c3d3, regionState=CLOSING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:20,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0f5137c718f1ce8c674c5a78d56c3d3, UNASSIGN because future has completed 2024-11-14T00:29:20,969 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T00:29:20,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:29:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T00:29:21,121 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:21,121 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T00:29:21,122 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing a0f5137c718f1ce8c674c5a78d56c3d3, disabling compactions & flushes 2024-11-14T00:29:21,122 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:21,122 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:21,122 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. after waiting 0 ms 2024-11-14T00:29:21,122 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:21,122 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing a0f5137c718f1ce8c674c5a78d56c3d3 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-14T00:29:21,131 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146321f266f3884b2dbf39555f11cd69b7_a0f5137c718f1ce8c674c5a78d56c3d3 is 50, key is test_row_0/A:col10/1731544160945/Put/seqid=0 2024-11-14T00:29:21,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742222_1398 (size=7374) 2024-11-14T00:29:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T00:29:21,548 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:21,552 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146321f266f3884b2dbf39555f11cd69b7_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146321f266f3884b2dbf39555f11cd69b7_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:21,553 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/1dbe8c44357546389d190abde1dbc001, store: [table=TestAcidGuarantees family=A region=a0f5137c718f1ce8c674c5a78d56c3d3] 2024-11-14T00:29:21,554 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/1dbe8c44357546389d190abde1dbc001 is 175, key is test_row_0/A:col10/1731544160945/Put/seqid=0 2024-11-14T00:29:21,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742223_1399 (size=13865) 2024-11-14T00:29:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T00:29:21,961 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/1dbe8c44357546389d190abde1dbc001 2024-11-14T00:29:21,970 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/ee0a022d91b04f3c87e185d01cf55023 is 50, key is test_row_0/B:col10/1731544160945/Put/seqid=0 2024-11-14T00:29:21,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742224_1400 (size=7415) 2024-11-14T00:29:21,975 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/ee0a022d91b04f3c87e185d01cf55023 2024-11-14T00:29:21,994 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/db60db48855f4514922d2e032ef6338b is 50, key is test_row_0/C:col10/1731544160945/Put/seqid=0 2024-11-14T00:29:22,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742225_1401 (size=7415) 2024-11-14T00:29:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T00:29:22,431 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/db60db48855f4514922d2e032ef6338b 2024-11-14T00:29:22,435 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/A/1dbe8c44357546389d190abde1dbc001 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/1dbe8c44357546389d190abde1dbc001 2024-11-14T00:29:22,440 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/1dbe8c44357546389d190abde1dbc001, entries=50, sequenceid=329, filesize=13.5 K 2024-11-14T00:29:22,441 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/B/ee0a022d91b04f3c87e185d01cf55023 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/ee0a022d91b04f3c87e185d01cf55023 2024-11-14T00:29:22,445 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/ee0a022d91b04f3c87e185d01cf55023, entries=50, sequenceid=329, filesize=7.2 K 2024-11-14T00:29:22,446 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/.tmp/C/db60db48855f4514922d2e032ef6338b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/db60db48855f4514922d2e032ef6338b 2024-11-14T00:29:22,455 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/db60db48855f4514922d2e032ef6338b, entries=50, sequenceid=329, filesize=7.2 K 2024-11-14T00:29:22,458 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for a0f5137c718f1ce8c674c5a78d56c3d3 in 1336ms, sequenceid=329, compaction requested=false 2024-11-14T00:29:22,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/0aacb2c43f2f4adf8cb5d99425a42dcf, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/edf2b44be63a447f973ccf1242f7bf3e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/12e02e0ee13f4cbfbee2fe9709b3ce54, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/c295b303296c4e0c894016d8aec16b57, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/9c6f6257c48541a1882f850863b8afa6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/7b1f79c675d84bdf8dd8e7ea98ebbb26, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/9b13fe17f74d4cbf88274c015f7091fd, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bff1406fa7f94e42bc8c84fd6a1d7a65, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/cd6b2eb2d6e0451fb286f15202effb04, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/8fc5275223eb4688af454ed20d47ed55, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/833ac59fe21442f7ba055964517c4830, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/11a10fa7e42b42d1b2210ddec758628a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/4882e969129e430896ecc411b6773052, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/6db57e3f36f049d39867907aa10c1bb2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/00d91f9fd7164577bbc8140db252c5bf, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/a79fc430ec9e41e4a85583b773c8ae28, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bcfc2b26e3f64f48bcdd9dc76bd25970, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bab549e0976944548176d6b16bc171fc, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/50878588dc954ee99f88f95b06723062, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/860744401ba94269aab4b9d8bb6a19cc, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/7e354f60103f4bbda8f700189afa369a] to archive 2024-11-14T00:29:22,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:29:22,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/0aacb2c43f2f4adf8cb5d99425a42dcf to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/0aacb2c43f2f4adf8cb5d99425a42dcf 2024-11-14T00:29:22,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/edf2b44be63a447f973ccf1242f7bf3e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/edf2b44be63a447f973ccf1242f7bf3e 2024-11-14T00:29:22,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/12e02e0ee13f4cbfbee2fe9709b3ce54 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/12e02e0ee13f4cbfbee2fe9709b3ce54 2024-11-14T00:29:22,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/c295b303296c4e0c894016d8aec16b57 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/c295b303296c4e0c894016d8aec16b57 2024-11-14T00:29:22,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/9c6f6257c48541a1882f850863b8afa6 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/9c6f6257c48541a1882f850863b8afa6 2024-11-14T00:29:22,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/7b1f79c675d84bdf8dd8e7ea98ebbb26 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/7b1f79c675d84bdf8dd8e7ea98ebbb26 2024-11-14T00:29:22,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/9b13fe17f74d4cbf88274c015f7091fd to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/9b13fe17f74d4cbf88274c015f7091fd 2024-11-14T00:29:22,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bff1406fa7f94e42bc8c84fd6a1d7a65 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bff1406fa7f94e42bc8c84fd6a1d7a65 2024-11-14T00:29:22,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/cd6b2eb2d6e0451fb286f15202effb04 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/cd6b2eb2d6e0451fb286f15202effb04 2024-11-14T00:29:22,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/8fc5275223eb4688af454ed20d47ed55 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/8fc5275223eb4688af454ed20d47ed55 2024-11-14T00:29:22,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/833ac59fe21442f7ba055964517c4830 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/833ac59fe21442f7ba055964517c4830 2024-11-14T00:29:22,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/11a10fa7e42b42d1b2210ddec758628a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/11a10fa7e42b42d1b2210ddec758628a 2024-11-14T00:29:22,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/4882e969129e430896ecc411b6773052 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/4882e969129e430896ecc411b6773052 2024-11-14T00:29:22,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/6db57e3f36f049d39867907aa10c1bb2 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/6db57e3f36f049d39867907aa10c1bb2 2024-11-14T00:29:22,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/00d91f9fd7164577bbc8140db252c5bf to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/00d91f9fd7164577bbc8140db252c5bf 2024-11-14T00:29:22,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/a79fc430ec9e41e4a85583b773c8ae28 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/a79fc430ec9e41e4a85583b773c8ae28 2024-11-14T00:29:22,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bcfc2b26e3f64f48bcdd9dc76bd25970 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bcfc2b26e3f64f48bcdd9dc76bd25970 2024-11-14T00:29:22,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bab549e0976944548176d6b16bc171fc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/bab549e0976944548176d6b16bc171fc 2024-11-14T00:29:22,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/50878588dc954ee99f88f95b06723062 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/50878588dc954ee99f88f95b06723062 2024-11-14T00:29:22,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/860744401ba94269aab4b9d8bb6a19cc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/860744401ba94269aab4b9d8bb6a19cc 2024-11-14T00:29:22,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/7e354f60103f4bbda8f700189afa369a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/7e354f60103f4bbda8f700189afa369a 2024-11-14T00:29:22,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/4a904103772b4f42af20514563936a66, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/fab0f924b3454995966f5784a9f7d29d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/f27b2bbb2c3a4b5bb04590544b5b1e1c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/f984b5c67c98406b9320d64d47600909, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/576f2aa43fb84120b358a58b3d4b08e0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/a5089d9d21ae437c932f019bdf603af0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/20f3a0b2734b45c4bd04492bd0496159, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/92cf16f4b2cf4f26adc0a53fa995255d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/3579ab65e5db427ea88d16f6f8b4bdd2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/6df26c50bc9b45a686534f80204b38ca, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/fc89dca19e03440bb8c19b01a7c00f76, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/6f7dc9bdef59440b882c51f9800fc4c5, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/19f67127540f42e69e8156df7e686fe7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/76d5fb431bcc4798bb548aaf0934bb36, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/514910f5300a44c7b78ca14690158280, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/1abd0bd89f0b477699e9f17c3b2cbe19, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/84a188d22c3f4c4c8b3ec79e9f1f725a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/c51ba0fee553455ea2b056b4bf7b7ab6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/5ec0d66a830b460fb40f8334e1d565d2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/8304806389a0475d803289ee8f931cf2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/338565bcfbc74779a61a596be75dd50a] to archive 2024-11-14T00:29:22,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:29:22,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/4a904103772b4f42af20514563936a66 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/4a904103772b4f42af20514563936a66 2024-11-14T00:29:22,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/fab0f924b3454995966f5784a9f7d29d to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/fab0f924b3454995966f5784a9f7d29d 2024-11-14T00:29:22,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/f27b2bbb2c3a4b5bb04590544b5b1e1c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/f27b2bbb2c3a4b5bb04590544b5b1e1c 2024-11-14T00:29:22,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/f984b5c67c98406b9320d64d47600909 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/f984b5c67c98406b9320d64d47600909 2024-11-14T00:29:22,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/576f2aa43fb84120b358a58b3d4b08e0 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/576f2aa43fb84120b358a58b3d4b08e0 2024-11-14T00:29:22,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/a5089d9d21ae437c932f019bdf603af0 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/a5089d9d21ae437c932f019bdf603af0 2024-11-14T00:29:22,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/20f3a0b2734b45c4bd04492bd0496159 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/20f3a0b2734b45c4bd04492bd0496159 2024-11-14T00:29:22,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/92cf16f4b2cf4f26adc0a53fa995255d to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/92cf16f4b2cf4f26adc0a53fa995255d 2024-11-14T00:29:22,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/3579ab65e5db427ea88d16f6f8b4bdd2 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/3579ab65e5db427ea88d16f6f8b4bdd2 2024-11-14T00:29:22,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/6df26c50bc9b45a686534f80204b38ca to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/6df26c50bc9b45a686534f80204b38ca 2024-11-14T00:29:22,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/fc89dca19e03440bb8c19b01a7c00f76 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/fc89dca19e03440bb8c19b01a7c00f76 2024-11-14T00:29:22,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/6f7dc9bdef59440b882c51f9800fc4c5 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/6f7dc9bdef59440b882c51f9800fc4c5 2024-11-14T00:29:22,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/19f67127540f42e69e8156df7e686fe7 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/19f67127540f42e69e8156df7e686fe7 2024-11-14T00:29:22,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/76d5fb431bcc4798bb548aaf0934bb36 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/76d5fb431bcc4798bb548aaf0934bb36 2024-11-14T00:29:22,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/514910f5300a44c7b78ca14690158280 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/514910f5300a44c7b78ca14690158280 2024-11-14T00:29:22,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/1abd0bd89f0b477699e9f17c3b2cbe19 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/1abd0bd89f0b477699e9f17c3b2cbe19 2024-11-14T00:29:22,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/84a188d22c3f4c4c8b3ec79e9f1f725a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/84a188d22c3f4c4c8b3ec79e9f1f725a 2024-11-14T00:29:22,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/c51ba0fee553455ea2b056b4bf7b7ab6 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/c51ba0fee553455ea2b056b4bf7b7ab6 2024-11-14T00:29:22,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/5ec0d66a830b460fb40f8334e1d565d2 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/5ec0d66a830b460fb40f8334e1d565d2 2024-11-14T00:29:22,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/8304806389a0475d803289ee8f931cf2 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/8304806389a0475d803289ee8f931cf2 2024-11-14T00:29:22,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/338565bcfbc74779a61a596be75dd50a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/338565bcfbc74779a61a596be75dd50a 2024-11-14T00:29:22,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/0a98f8c868444ee2bb5e6eb0e09cf95f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/dd6c27d5365e4f8e86bc36d1cfb030e2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/71394d64981a4cc3a3486ecfe1ec42f0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/62d210dacaca41b0b8752beeff78b378, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/681bdf8289b5420391cc031e79a981fc, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/2428ba8b9bca45fc8a628208cfa29c42, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a839975e18244716b299abd5cc3ca768, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/976a697a75d74a9998eea6e2c90c6499, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/cc5b90f33bdd4e78af11adef0c5266e7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/c526db6f9e5b4ad0be935f979ca3c48a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/dae1f244a9024f62ac9929b254711f39, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/95bb4438c6ea4ddf95ec2bb4541e17e0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a3f854d5c4f944129c5c8ab3aeb7228e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/d3b08719d29944948ffb8390bdaa23cf, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/5017a2ca4f164d03b81691cd02c19df0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/494f8e9cdfb74b0cbe61e68c441be91b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/b1d207c8cf144348ba41f66d0bffb676, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/0aa5576f57054024b8479711b7aa67eb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a551e386985442e88d69d94339133c24, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/8695c3a0a57642a1ae13d323c20a9a98, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/90f3c96b5f39401783e90d3ffd3639ac] to archive 2024-11-14T00:29:22,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:29:22,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/0a98f8c868444ee2bb5e6eb0e09cf95f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/0a98f8c868444ee2bb5e6eb0e09cf95f 2024-11-14T00:29:22,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/dd6c27d5365e4f8e86bc36d1cfb030e2 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/dd6c27d5365e4f8e86bc36d1cfb030e2 2024-11-14T00:29:22,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/71394d64981a4cc3a3486ecfe1ec42f0 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/71394d64981a4cc3a3486ecfe1ec42f0 2024-11-14T00:29:22,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/62d210dacaca41b0b8752beeff78b378 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/62d210dacaca41b0b8752beeff78b378 2024-11-14T00:29:22,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/681bdf8289b5420391cc031e79a981fc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/681bdf8289b5420391cc031e79a981fc 2024-11-14T00:29:22,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/2428ba8b9bca45fc8a628208cfa29c42 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/2428ba8b9bca45fc8a628208cfa29c42 2024-11-14T00:29:22,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a839975e18244716b299abd5cc3ca768 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a839975e18244716b299abd5cc3ca768 2024-11-14T00:29:22,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/976a697a75d74a9998eea6e2c90c6499 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/976a697a75d74a9998eea6e2c90c6499 2024-11-14T00:29:22,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/cc5b90f33bdd4e78af11adef0c5266e7 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/cc5b90f33bdd4e78af11adef0c5266e7 2024-11-14T00:29:22,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/c526db6f9e5b4ad0be935f979ca3c48a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/c526db6f9e5b4ad0be935f979ca3c48a 2024-11-14T00:29:22,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/dae1f244a9024f62ac9929b254711f39 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/dae1f244a9024f62ac9929b254711f39 2024-11-14T00:29:22,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/95bb4438c6ea4ddf95ec2bb4541e17e0 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/95bb4438c6ea4ddf95ec2bb4541e17e0 2024-11-14T00:29:22,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a3f854d5c4f944129c5c8ab3aeb7228e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a3f854d5c4f944129c5c8ab3aeb7228e 2024-11-14T00:29:22,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/d3b08719d29944948ffb8390bdaa23cf to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/d3b08719d29944948ffb8390bdaa23cf 2024-11-14T00:29:22,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/5017a2ca4f164d03b81691cd02c19df0 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/5017a2ca4f164d03b81691cd02c19df0 2024-11-14T00:29:22,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/494f8e9cdfb74b0cbe61e68c441be91b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/494f8e9cdfb74b0cbe61e68c441be91b 2024-11-14T00:29:22,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/b1d207c8cf144348ba41f66d0bffb676 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/b1d207c8cf144348ba41f66d0bffb676 2024-11-14T00:29:22,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/0aa5576f57054024b8479711b7aa67eb to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/0aa5576f57054024b8479711b7aa67eb 2024-11-14T00:29:22,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a551e386985442e88d69d94339133c24 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/a551e386985442e88d69d94339133c24 2024-11-14T00:29:22,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/8695c3a0a57642a1ae13d323c20a9a98 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/8695c3a0a57642a1ae13d323c20a9a98 2024-11-14T00:29:22,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/90f3c96b5f39401783e90d3ffd3639ac to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/90f3c96b5f39401783e90d3ffd3639ac 2024-11-14T00:29:22,671 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/recovered.edits/332.seqid, newMaxSeqId=332, maxSeqId=4 2024-11-14T00:29:22,671 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. 2024-11-14T00:29:22,672 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for a0f5137c718f1ce8c674c5a78d56c3d3: Waiting for close lock at 1731544161122Running coprocessor pre-close hooks at 1731544161122Disabling compacts and flushes for region at 1731544161122Disabling writes for close at 1731544161122Obtaining lock to block concurrent updates at 1731544161122Preparing flush snapshotting stores in a0f5137c718f1ce8c674c5a78d56c3d3 at 1731544161122Finished memstore snapshotting TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731544161122Flushing stores of TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3. at 1731544161123 (+1 ms)Flushing a0f5137c718f1ce8c674c5a78d56c3d3/A: creating writer at 1731544161123Flushing mob file a0f5137c718f1ce8c674c5a78d56c3d3/A: appending metadata at 1731544161131 (+8 ms)Flushing mob file a0f5137c718f1ce8c674c5a78d56c3d3/A: closing flushed file at 1731544161131Flushing a0f5137c718f1ce8c674c5a78d56c3d3/A: appending metadata at 1731544161553 (+422 ms)Flushing a0f5137c718f1ce8c674c5a78d56c3d3/A: closing flushed file at 1731544161553Flushing a0f5137c718f1ce8c674c5a78d56c3d3/B: creating writer at 1731544161967 (+414 ms)Flushing a0f5137c718f1ce8c674c5a78d56c3d3/B: appending metadata at 1731544161970 (+3 ms)Flushing a0f5137c718f1ce8c674c5a78d56c3d3/B: closing flushed file at 1731544161970Flushing a0f5137c718f1ce8c674c5a78d56c3d3/C: creating writer at 1731544161987 (+17 ms)Flushing a0f5137c718f1ce8c674c5a78d56c3d3/C: appending metadata at 1731544161993 (+6 ms)Flushing a0f5137c718f1ce8c674c5a78d56c3d3/C: closing flushed file at 1731544161993Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ca8103b: reopening flushed file at 1731544162434 (+441 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4feccfb1: reopening flushed file at 1731544162440 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bf3ee4f: reopening flushed file at 1731544162445 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for a0f5137c718f1ce8c674c5a78d56c3d3 in 1336ms, sequenceid=329, compaction requested=false at 1731544162458 (+13 ms)Writing region close event to WAL at 1731544162668 (+210 ms)Running coprocessor post-close hooks at 1731544162671 (+3 ms)Closed at 1731544162671 2024-11-14T00:29:22,673 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:22,673 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=a0f5137c718f1ce8c674c5a78d56c3d3, regionState=CLOSED 2024-11-14T00:29:22,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:29:22,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-14T00:29:22,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure a0f5137c718f1ce8c674c5a78d56c3d3, server=63baa11b75f3,33199,1731544045387 in 1.7060 sec 2024-11-14T00:29:22,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-14T00:29:22,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0f5137c718f1ce8c674c5a78d56c3d3, UNASSIGN in 1.7130 sec 2024-11-14T00:29:22,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-14T00:29:22,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7200 sec 2024-11-14T00:29:22,681 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544162681"}]},"ts":"1731544162681"} 2024-11-14T00:29:22,682 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T00:29:22,682 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T00:29:22,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7340 sec 2024-11-14T00:29:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T00:29:23,105 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:23,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-14T00:29:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:23,107 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T00:29:23,108 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:23,111 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,112 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/recovered.edits] 2024-11-14T00:29:23,114 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/1dbe8c44357546389d190abde1dbc001 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/1dbe8c44357546389d190abde1dbc001 2024-11-14T00:29:23,115 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/3732982f85c5458389d12597a83d4543 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/A/3732982f85c5458389d12597a83d4543 2024-11-14T00:29:23,117 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/2c52826e578c4afcadca3e1bfd5bfcc5 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/2c52826e578c4afcadca3e1bfd5bfcc5 2024-11-14T00:29:23,117 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/ee0a022d91b04f3c87e185d01cf55023 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/B/ee0a022d91b04f3c87e185d01cf55023 2024-11-14T00:29:23,119 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/2789fde2c0ac46d89e0e84f1115ca541 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/2789fde2c0ac46d89e0e84f1115ca541 2024-11-14T00:29:23,120 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/db60db48855f4514922d2e032ef6338b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/C/db60db48855f4514922d2e032ef6338b 2024-11-14T00:29:23,122 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/recovered.edits/332.seqid to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3/recovered.edits/332.seqid 2024-11-14T00:29:23,122 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,122 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T00:29:23,122 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T00:29:23,123 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T00:29:23,125 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114338132fb0680427c821e07350399e477_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114338132fb0680427c821e07350399e477_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,126 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111444dae1f1c19c4041804eb256c44f38c7_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111444dae1f1c19c4041804eb256c44f38c7_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,127 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145e3309f4aabd4e2ca041dd3fa43f4140_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145e3309f4aabd4e2ca041dd3fa43f4140_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,128 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114601279a36d154cee8799b103e1dbf281_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114601279a36d154cee8799b103e1dbf281_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,128 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146321f266f3884b2dbf39555f11cd69b7_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146321f266f3884b2dbf39555f11cd69b7_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,129 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111464a183947cc44e94a956943652e766a3_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111464a183947cc44e94a956943652e766a3_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,130 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111495d741eb29b44f5cb2ef2d1827d2cb2f_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111495d741eb29b44f5cb2ef2d1827d2cb2f_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,131 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a48a05981d02412eb958c808970b47d4_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a48a05981d02412eb958c808970b47d4_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,132 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b9ef57611f0647bb9c1a0739c4114c7f_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b9ef57611f0647bb9c1a0739c4114c7f_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,133 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c21ab6af74f6417983b73fbe313814a7_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c21ab6af74f6417983b73fbe313814a7_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,134 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cb9d49c9d518450f962041e32bc16a43_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cb9d49c9d518450f962041e32bc16a43_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,134 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cf525681f7d648aface80c3060976ea0_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114cf525681f7d648aface80c3060976ea0_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,135 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dbbe89f38ec340a5957ee96fb1e9f3e1_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114dbbe89f38ec340a5957ee96fb1e9f3e1_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,136 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114df1d79d46c594b5b88de0acd9b9f285c_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114df1d79d46c594b5b88de0acd9b9f285c_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,137 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb6a0dc3caf64ffd9a12f33f8396f011_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb6a0dc3caf64ffd9a12f33f8396f011_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,137 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f15cd7027311490fa4effb3675cf9a6d_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f15cd7027311490fa4effb3675cf9a6d_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,138 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f2b6b71149cb4dd9954cb7145a933e40_a0f5137c718f1ce8c674c5a78d56c3d3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f2b6b71149cb4dd9954cb7145a933e40_a0f5137c718f1ce8c674c5a78d56c3d3 2024-11-14T00:29:23,138 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T00:29:23,140 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:23,142 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T00:29:23,143 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T00:29:23,144 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:23,144 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T00:29:23,145 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731544163144"}]},"ts":"9223372036854775807"} 2024-11-14T00:29:23,146 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T00:29:23,146 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a0f5137c718f1ce8c674c5a78d56c3d3, NAME => 'TestAcidGuarantees,,1731544135232.a0f5137c718f1ce8c674c5a78d56c3d3.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T00:29:23,146 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T00:29:23,147 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731544163146"}]},"ts":"9223372036854775807"} 2024-11-14T00:29:23,149 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T00:29:23,150 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:23,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-14T00:29:23,207 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T00:29:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T00:29:23,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T00:29:23,218 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:23,224 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1906822607_22 at /127.0.0.1:45538 [Waiting for operation #959] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1495897995_22 at /127.0.0.1:45524 [Waiting for operation #919] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1495897995_22 at /127.0.0.1:38098 [Waiting for operation #1371] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1906822607_22 at /127.0.0.1:54016 [Waiting for operation #1232] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=463 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1272 (was 1190) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2748 (was 3123) 2024-11-14T00:29:23,231 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240, OpenFileDescriptor=463, MaxFileDescriptor=1048576, SystemLoadAverage=1272, ProcessCount=11, AvailableMemoryMB=2748 2024-11-14T00:29:23,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T00:29:23,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T00:29:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:23,234 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T00:29:23,235 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:23,235 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-14T00:29:23,235 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T00:29:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-14T00:29:23,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742226_1402 (size=959) 2024-11-14T00:29:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-14T00:29:23,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-14T00:29:23,642 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c 2024-11-14T00:29:23,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742227_1403 (size=53) 2024-11-14T00:29:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-14T00:29:24,084 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:29:24,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 93f27665e92d07bc413648d25770240d, disabling compactions & flushes 2024-11-14T00:29:24,085 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:24,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:24,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. after waiting 0 ms 2024-11-14T00:29:24,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:24,085 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:24,085 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 93f27665e92d07bc413648d25770240d: Waiting for close lock at 1731544164085Disabling compacts and flushes for region at 1731544164085Disabling writes for close at 1731544164085Writing region close event to WAL at 1731544164085Closed at 1731544164085 2024-11-14T00:29:24,086 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T00:29:24,086 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731544164086"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731544164086"}]},"ts":"1731544164086"} 2024-11-14T00:29:24,088 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T00:29:24,092 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T00:29:24,092 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544164092"}]},"ts":"1731544164092"} 2024-11-14T00:29:24,098 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T00:29:24,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=93f27665e92d07bc413648d25770240d, ASSIGN}] 2024-11-14T00:29:24,100 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=93f27665e92d07bc413648d25770240d, ASSIGN 2024-11-14T00:29:24,100 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=93f27665e92d07bc413648d25770240d, ASSIGN; state=OFFLINE, location=63baa11b75f3,33199,1731544045387; forceNewPlan=false, retain=false 2024-11-14T00:29:24,251 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=93f27665e92d07bc413648d25770240d, regionState=OPENING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:24,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=93f27665e92d07bc413648d25770240d, ASSIGN because future has completed 2024-11-14T00:29:24,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:29:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-14T00:29:24,408 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:24,408 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} 2024-11-14T00:29:24,409 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:24,409 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:29:24,409 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:24,409 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:24,410 INFO [StoreOpener-93f27665e92d07bc413648d25770240d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:24,411 INFO [StoreOpener-93f27665e92d07bc413648d25770240d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93f27665e92d07bc413648d25770240d columnFamilyName A 2024-11-14T00:29:24,411 DEBUG [StoreOpener-93f27665e92d07bc413648d25770240d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:24,412 INFO [StoreOpener-93f27665e92d07bc413648d25770240d-1 {}] regionserver.HStore(327): Store=93f27665e92d07bc413648d25770240d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:29:24,412 INFO [StoreOpener-93f27665e92d07bc413648d25770240d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:24,415 INFO [StoreOpener-93f27665e92d07bc413648d25770240d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93f27665e92d07bc413648d25770240d columnFamilyName B 2024-11-14T00:29:24,415 DEBUG [StoreOpener-93f27665e92d07bc413648d25770240d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:24,415 INFO [StoreOpener-93f27665e92d07bc413648d25770240d-1 {}] regionserver.HStore(327): Store=93f27665e92d07bc413648d25770240d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:29:24,416 INFO [StoreOpener-93f27665e92d07bc413648d25770240d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:24,417 INFO [StoreOpener-93f27665e92d07bc413648d25770240d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93f27665e92d07bc413648d25770240d columnFamilyName C 2024-11-14T00:29:24,417 DEBUG [StoreOpener-93f27665e92d07bc413648d25770240d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:24,418 INFO [StoreOpener-93f27665e92d07bc413648d25770240d-1 {}] regionserver.HStore(327): Store=93f27665e92d07bc413648d25770240d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:29:24,418 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:24,419 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d 2024-11-14T00:29:24,419 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d 2024-11-14T00:29:24,420 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:24,420 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:24,420 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T00:29:24,422 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:24,424 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T00:29:24,424 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 93f27665e92d07bc413648d25770240d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68208026, jitterRate=0.016378790140151978}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T00:29:24,424 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:24,425 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 93f27665e92d07bc413648d25770240d: Running coprocessor pre-open hook at 1731544164409Writing region info on filesystem at 1731544164409Initializing all the Stores at 1731544164410 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544164410Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544164410Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544164410Cleaning up temporary data from old regions at 1731544164420 (+10 ms)Running coprocessor post-open hooks at 1731544164424 (+4 ms)Region opened successfully at 1731544164425 (+1 ms) 2024-11-14T00:29:24,425 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., pid=114, masterSystemTime=1731544164405 2024-11-14T00:29:24,427 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:24,427 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:24,428 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=93f27665e92d07bc413648d25770240d, regionState=OPEN, openSeqNum=2, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:24,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:29:24,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-14T00:29:24,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 in 177 msec 2024-11-14T00:29:24,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-14T00:29:24,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=93f27665e92d07bc413648d25770240d, ASSIGN in 333 msec 2024-11-14T00:29:24,434 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T00:29:24,434 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544164434"}]},"ts":"1731544164434"} 2024-11-14T00:29:24,435 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T00:29:24,436 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T00:29:24,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2030 sec 2024-11-14T00:29:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-14T00:29:25,393 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:25,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e884694, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:25,394 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:25,395 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23f3b3a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:25,397 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:25,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3029732d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,401 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:25,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:25,404 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:25,405 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T00:29:25,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 63baa11b75f3,36271,1731544044347 2024-11-14T00:29:25,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4bac4be3 2024-11-14T00:29:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T00:29:25,408 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T00:29:25,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e7e3294, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,412 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:25,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:25,414 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@510c2ddb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:25,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:25,416 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:25,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763fc8db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@588f482, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,418 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:25,418 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:25,432 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:25,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44e76a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:25,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:25,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:25,435 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:25,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29ca9d66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65da83b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:25,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:25,440 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:25,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b11bffa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:25,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:25,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:25,445 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:25,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d517e57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@780b99fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,448 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:25,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:25,453 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c91e920, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:25,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:25,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:25,455 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c8faf14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683026da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,460 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:25,461 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:25,465 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3157d9d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:25,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:25,467 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46f8bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@733fcf91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,473 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:25,473 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:25,475 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:25,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:25,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:25,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@615af8e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:25,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:25,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:25,480 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:25,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d35c547, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5921fce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,483 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:25,483 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:25,486 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:25,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66193df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:25,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:25,488 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:25,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5227c511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b52d1e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,492 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:25,493 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:25,494 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a28697c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:25,495 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d0f6c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25b9c52b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,497 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:25,497 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:25,498 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:25,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:25,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13907c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:25,501 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:25,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34746180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ef3354, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,505 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:25,505 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:25,506 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54064370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:25,508 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b5e0c07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:25,509 DEBUG [Thread-1830 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:25,509 DEBUG [Thread-1839 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:25,509 DEBUG [Thread-1843 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:25,510 DEBUG [Thread-1826 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:25,511 DEBUG [Thread-1832 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:25,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:25,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:25,512 DEBUG [Thread-1832 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:25,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:25,512 DEBUG [Thread-1832 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:25,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:25,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:25,514 DEBUG [Thread-1837 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:29:25,514 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:29:25,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:29:25,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:29:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-14T00:29:25,516 DEBUG [Thread-1837 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:25,517 DEBUG [Thread-1837 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:25,517 DEBUG [Thread-1841 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:25,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:29:25,518 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:29:25,518 DEBUG [Thread-1834 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:25,519 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:29:25,519 DEBUG [Thread-1845 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:25,519 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:29:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T00:29:25,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:25,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:25,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:29:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:29:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:25,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:25,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:25,522 DEBUG [Thread-1828 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:25,523 DEBUG [Thread-1828 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:25,524 DEBUG [Thread-1828 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:29:25,528 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:25,530 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:25,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:25,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:29:25,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:25,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:29:25,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:25,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2] 2024-11-14T00:29:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:25,555 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:25,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T00:29:25,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/647dd056552d45a6bc120126928a322c is 50, key is test_row_0/A:col10/1731544165554/Put/seqid=0 2024-11-14T00:29:25,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742228_1404 (size=12001) 2024-11-14T00:29:25,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544225607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:25,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544225610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T00:29:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:25,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:25,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544225614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:25,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544225619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:25,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544225625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:25,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:25,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T00:29:25,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:25,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:25,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:25,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:25,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:25,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:25,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544225736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:25,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544225739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:25,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544225746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:25,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544225746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:25,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544225747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:25,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T00:29:25,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T00:29:25,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:25,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:25,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:25,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:25,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:25,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:25,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544225960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:25,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:25,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544225961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:25,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:25,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:25,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544225961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:25,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544225965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:25,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544225965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:25,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T00:29:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:25,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:25,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:25,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:25,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/647dd056552d45a6bc120126928a322c 2024-11-14T00:29:26,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/74ce38f4466b4d0aa317c2753125b7ed is 50, key is test_row_0/B:col10/1731544165554/Put/seqid=0 2024-11-14T00:29:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T00:29:26,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742229_1405 (size=12001) 2024-11-14T00:29:26,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T00:29:26,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:26,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:26,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544226276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:26,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544226277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544226277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:26,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544226277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:26,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544226277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:26,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T00:29:26,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:26,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:26,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:26,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T00:29:26,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:26,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:26,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:26,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/74ce38f4466b4d0aa317c2753125b7ed 2024-11-14T00:29:26,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T00:29:26,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:26,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:26,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:26,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T00:29:26,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/e55111466f2d486ca4f8dde2f226d545 is 50, key is test_row_0/C:col10/1731544165554/Put/seqid=0 2024-11-14T00:29:26,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742230_1406 (size=12001) 2024-11-14T00:29:26,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T00:29:26,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:26,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:26,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:26,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544226790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:26,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544226796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:26,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544226796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:26,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544226796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:26,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544226808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:26,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:26,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T00:29:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:26,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:27,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T00:29:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:27,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:27,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:27,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/e55111466f2d486ca4f8dde2f226d545 2024-11-14T00:29:27,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/647dd056552d45a6bc120126928a322c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/647dd056552d45a6bc120126928a322c 2024-11-14T00:29:27,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/647dd056552d45a6bc120126928a322c, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T00:29:27,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/74ce38f4466b4d0aa317c2753125b7ed as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/74ce38f4466b4d0aa317c2753125b7ed 2024-11-14T00:29:27,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/74ce38f4466b4d0aa317c2753125b7ed, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T00:29:27,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/e55111466f2d486ca4f8dde2f226d545 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/e55111466f2d486ca4f8dde2f226d545 2024-11-14T00:29:27,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/e55111466f2d486ca4f8dde2f226d545, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T00:29:27,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 93f27665e92d07bc413648d25770240d in 1614ms, sequenceid=12, compaction requested=false 2024-11-14T00:29:27,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:27,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T00:29:27,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:27,262 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T00:29:27,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/08d2e7030faf415081c74f9fe37ad7bd is 50, key is test_row_0/A:col10/1731544165604/Put/seqid=0 2024-11-14T00:29:27,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742231_1407 (size=12001) 2024-11-14T00:29:27,323 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/08d2e7030faf415081c74f9fe37ad7bd 2024-11-14T00:29:27,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/2ae2a3033e484130a141209bfa6cbc7f is 50, key is test_row_0/B:col10/1731544165604/Put/seqid=0 2024-11-14T00:29:27,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742232_1408 (size=12001) 2024-11-14T00:29:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T00:29:27,817 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/2ae2a3033e484130a141209bfa6cbc7f 2024-11-14T00:29:27,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:27,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/62fee99beafa49659201d47150e9fe4e is 50, key is test_row_0/C:col10/1731544165604/Put/seqid=0 2024-11-14T00:29:27,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544227840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:27,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544227848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:27,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544227849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:27,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:27,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:27,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544227859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:27,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544227859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:27,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742233_1409 (size=12001) 2024-11-14T00:29:27,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:27,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:27,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:27,874 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/62fee99beafa49659201d47150e9fe4e 2024-11-14T00:29:27,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/08d2e7030faf415081c74f9fe37ad7bd as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/08d2e7030faf415081c74f9fe37ad7bd 2024-11-14T00:29:27,913 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/08d2e7030faf415081c74f9fe37ad7bd, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T00:29:27,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/2ae2a3033e484130a141209bfa6cbc7f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/2ae2a3033e484130a141209bfa6cbc7f 2024-11-14T00:29:27,921 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/2ae2a3033e484130a141209bfa6cbc7f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T00:29:27,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/62fee99beafa49659201d47150e9fe4e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/62fee99beafa49659201d47150e9fe4e 2024-11-14T00:29:27,942 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/62fee99beafa49659201d47150e9fe4e, entries=150, sequenceid=37, filesize=11.7 K 2024-11-14T00:29:27,944 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 93f27665e92d07bc413648d25770240d in 683ms, sequenceid=37, compaction requested=false 2024-11-14T00:29:27,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:27,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:27,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-14T00:29:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-14T00:29:27,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-14T00:29:27,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4270 sec 2024-11-14T00:29:27,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.4350 sec 2024-11-14T00:29:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:27,981 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:27,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T00:29:28,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/26322032678847c1b679cf61456b37c7 is 50, key is test_row_0/A:col10/1731544167981/Put/seqid=0 2024-11-14T00:29:28,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742234_1410 (size=16681) 2024-11-14T00:29:28,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/26322032678847c1b679cf61456b37c7 2024-11-14T00:29:28,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/ce5dfb5efd4847b3a9daeb7a9b1b4197 is 50, key is test_row_0/B:col10/1731544167981/Put/seqid=0 2024-11-14T00:29:28,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544228037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544228039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544228044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544228046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544228047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742235_1411 (size=12001) 2024-11-14T00:29:28,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/ce5dfb5efd4847b3a9daeb7a9b1b4197 2024-11-14T00:29:28,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/4040c9895d2b414ea239872203fb847d is 50, key is test_row_0/C:col10/1731544167981/Put/seqid=0 2024-11-14T00:29:28,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742236_1412 (size=12001) 2024-11-14T00:29:28,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/4040c9895d2b414ea239872203fb847d 2024-11-14T00:29:28,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/26322032678847c1b679cf61456b37c7 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/26322032678847c1b679cf61456b37c7 2024-11-14T00:29:28,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544228165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544228166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/26322032678847c1b679cf61456b37c7, entries=250, sequenceid=52, filesize=16.3 K 2024-11-14T00:29:28,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/ce5dfb5efd4847b3a9daeb7a9b1b4197 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/ce5dfb5efd4847b3a9daeb7a9b1b4197 2024-11-14T00:29:28,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/ce5dfb5efd4847b3a9daeb7a9b1b4197, entries=150, sequenceid=52, filesize=11.7 K 2024-11-14T00:29:28,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/4040c9895d2b414ea239872203fb847d as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4040c9895d2b414ea239872203fb847d 2024-11-14T00:29:28,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4040c9895d2b414ea239872203fb847d, entries=150, sequenceid=52, filesize=11.7 K 2024-11-14T00:29:28,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 93f27665e92d07bc413648d25770240d in 211ms, sequenceid=52, compaction requested=true 2024-11-14T00:29:28,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:28,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:28,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:28,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:28,193 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:28,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:28,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:28,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:29:28,193 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:28,194 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:28,194 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:28,194 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/A is initiating minor compaction (all files) 2024-11-14T00:29:28,194 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/B is initiating minor compaction (all files) 2024-11-14T00:29:28,194 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/B in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:28,194 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/A in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:28,194 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/74ce38f4466b4d0aa317c2753125b7ed, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/2ae2a3033e484130a141209bfa6cbc7f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/ce5dfb5efd4847b3a9daeb7a9b1b4197] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=35.2 K 2024-11-14T00:29:28,194 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/647dd056552d45a6bc120126928a322c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/08d2e7030faf415081c74f9fe37ad7bd, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/26322032678847c1b679cf61456b37c7] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=39.7 K 2024-11-14T00:29:28,194 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74ce38f4466b4d0aa317c2753125b7ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731544165529 2024-11-14T00:29:28,194 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 647dd056552d45a6bc120126928a322c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731544165529 2024-11-14T00:29:28,195 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 08d2e7030faf415081c74f9fe37ad7bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731544165604 2024-11-14T00:29:28,195 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ae2a3033e484130a141209bfa6cbc7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731544165604 2024-11-14T00:29:28,195 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 26322032678847c1b679cf61456b37c7, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544167840 2024-11-14T00:29:28,195 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce5dfb5efd4847b3a9daeb7a9b1b4197, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544167840 2024-11-14T00:29:28,200 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:28,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T00:29:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:28,207 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:28,207 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/186c49d849b94e6c864295f764248c5b is 50, key is test_row_0/B:col10/1731544167981/Put/seqid=0 2024-11-14T00:29:28,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/b4fc5f8b09cb4ed7b5972dad29d7dfe8 is 50, key is test_row_0/A:col10/1731544168198/Put/seqid=0 2024-11-14T00:29:28,219 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T00:29:28,221 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#A#compaction#344 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:28,221 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/f5c0b1799a8642948bff3cb80e498611 is 50, key is test_row_0/A:col10/1731544167981/Put/seqid=0 2024-11-14T00:29:28,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742237_1413 (size=12104) 2024-11-14T00:29:28,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742238_1414 (size=16681) 2024-11-14T00:29:28,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544228279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544228281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,313 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/186c49d849b94e6c864295f764248c5b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/186c49d849b94e6c864295f764248c5b 2024-11-14T00:29:28,320 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/B of 93f27665e92d07bc413648d25770240d into 186c49d849b94e6c864295f764248c5b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:28,320 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:28,320 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/B, priority=13, startTime=1731544168193; duration=0sec 2024-11-14T00:29:28,320 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:28,320 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:B 2024-11-14T00:29:28,320 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:28,323 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:28,324 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/C is initiating minor compaction (all files) 2024-11-14T00:29:28,324 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/C in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:28,324 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/e55111466f2d486ca4f8dde2f226d545, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/62fee99beafa49659201d47150e9fe4e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4040c9895d2b414ea239872203fb847d] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=35.2 K 2024-11-14T00:29:28,326 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting e55111466f2d486ca4f8dde2f226d545, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731544165529 2024-11-14T00:29:28,326 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62fee99beafa49659201d47150e9fe4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731544165604 2024-11-14T00:29:28,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742239_1415 (size=12104) 2024-11-14T00:29:28,327 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4040c9895d2b414ea239872203fb847d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544167840 2024-11-14T00:29:28,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544228309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,341 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/f5c0b1799a8642948bff3cb80e498611 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/f5c0b1799a8642948bff3cb80e498611 2024-11-14T00:29:28,346 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/A of 93f27665e92d07bc413648d25770240d into f5c0b1799a8642948bff3cb80e498611(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:28,346 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:28,346 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/A, priority=13, startTime=1731544168193; duration=0sec 2024-11-14T00:29:28,346 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:28,346 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:A 2024-11-14T00:29:28,362 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#C#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:28,363 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/27e1925ef9664e5ab6958d5e82d1d346 is 50, key is test_row_0/C:col10/1731544167981/Put/seqid=0 2024-11-14T00:29:28,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544228381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544228381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742240_1416 (size=12104) 2024-11-14T00:29:28,423 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/27e1925ef9664e5ab6958d5e82d1d346 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/27e1925ef9664e5ab6958d5e82d1d346 2024-11-14T00:29:28,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544228415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544228416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,438 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/C of 93f27665e92d07bc413648d25770240d into 27e1925ef9664e5ab6958d5e82d1d346(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:28,438 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:28,438 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/C, priority=13, startTime=1731544168193; duration=0sec 2024-11-14T00:29:28,439 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:28,439 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:C 2024-11-14T00:29:28,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544228448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,507 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/63baa11b75f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/647dd056552d45a6bc120126928a322c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/08d2e7030faf415081c74f9fe37ad7bd, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/26322032678847c1b679cf61456b37c7] to archive 2024-11-14T00:29:28,516 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/63baa11b75f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:29:28,532 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/63baa11b75f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/647dd056552d45a6bc120126928a322c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/647dd056552d45a6bc120126928a322c 2024-11-14T00:29:28,533 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/63baa11b75f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/08d2e7030faf415081c74f9fe37ad7bd to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/08d2e7030faf415081c74f9fe37ad7bd 2024-11-14T00:29:28,535 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/63baa11b75f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/26322032678847c1b679cf61456b37c7 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/26322032678847c1b679cf61456b37c7 2024-11-14T00:29:28,541 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/63baa11b75f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/74ce38f4466b4d0aa317c2753125b7ed, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/2ae2a3033e484130a141209bfa6cbc7f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/ce5dfb5efd4847b3a9daeb7a9b1b4197] to archive 2024-11-14T00:29:28,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/63baa11b75f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:29:28,544 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/63baa11b75f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/74ce38f4466b4d0aa317c2753125b7ed to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/74ce38f4466b4d0aa317c2753125b7ed 2024-11-14T00:29:28,545 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/63baa11b75f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/2ae2a3033e484130a141209bfa6cbc7f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/2ae2a3033e484130a141209bfa6cbc7f 2024-11-14T00:29:28,546 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/63baa11b75f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/ce5dfb5efd4847b3a9daeb7a9b1b4197 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/ce5dfb5efd4847b3a9daeb7a9b1b4197 2024-11-14T00:29:28,547 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/63baa11b75f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/e55111466f2d486ca4f8dde2f226d545, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/62fee99beafa49659201d47150e9fe4e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4040c9895d2b414ea239872203fb847d] to archive 2024-11-14T00:29:28,548 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/63baa11b75f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:29:28,550 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/63baa11b75f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/e55111466f2d486ca4f8dde2f226d545 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/e55111466f2d486ca4f8dde2f226d545 2024-11-14T00:29:28,552 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/63baa11b75f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/62fee99beafa49659201d47150e9fe4e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/62fee99beafa49659201d47150e9fe4e 2024-11-14T00:29:28,554 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/63baa11b75f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4040c9895d2b414ea239872203fb847d to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4040c9895d2b414ea239872203fb847d 2024-11-14T00:29:28,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544228646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544228646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544228668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544228705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/b4fc5f8b09cb4ed7b5972dad29d7dfe8 2024-11-14T00:29:28,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/16a67c68d3ac4265a51a708399f13e44 is 50, key is test_row_0/B:col10/1731544168198/Put/seqid=0 2024-11-14T00:29:28,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544228716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:28,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:28,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:28,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742241_1417 (size=12001) 2024-11-14T00:29:28,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/16a67c68d3ac4265a51a708399f13e44 2024-11-14T00:29:28,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/272f1fdb574f47fc80f78c7deecb88ca is 50, key is test_row_0/C:col10/1731544168198/Put/seqid=0 2024-11-14T00:29:28,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742242_1418 (size=12001) 2024-11-14T00:29:28,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/272f1fdb574f47fc80f78c7deecb88ca 2024-11-14T00:29:28,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/b4fc5f8b09cb4ed7b5972dad29d7dfe8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/b4fc5f8b09cb4ed7b5972dad29d7dfe8 2024-11-14T00:29:28,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/b4fc5f8b09cb4ed7b5972dad29d7dfe8, entries=250, sequenceid=75, filesize=16.3 K 2024-11-14T00:29:28,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/16a67c68d3ac4265a51a708399f13e44 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/16a67c68d3ac4265a51a708399f13e44 2024-11-14T00:29:28,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/16a67c68d3ac4265a51a708399f13e44, entries=150, sequenceid=75, filesize=11.7 K 2024-11-14T00:29:28,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/272f1fdb574f47fc80f78c7deecb88ca as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/272f1fdb574f47fc80f78c7deecb88ca 2024-11-14T00:29:28,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/272f1fdb574f47fc80f78c7deecb88ca, entries=150, sequenceid=75, filesize=11.7 K 2024-11-14T00:29:28,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 93f27665e92d07bc413648d25770240d in 753ms, sequenceid=75, compaction requested=false 2024-11-14T00:29:28,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:28,961 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:28,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T00:29:28,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/2fa7d66fbb0743e28af5eef6aeb18bf0 is 50, key is test_row_0/A:col10/1731544168278/Put/seqid=0 2024-11-14T00:29:29,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742243_1419 (size=14341) 2024-11-14T00:29:29,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/2fa7d66fbb0743e28af5eef6aeb18bf0 2024-11-14T00:29:29,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/2af7fffbdf1e4635be6e6b3d16132b0c is 50, key is test_row_0/B:col10/1731544168278/Put/seqid=0 2024-11-14T00:29:29,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742244_1420 (size=12001) 2024-11-14T00:29:29,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/2af7fffbdf1e4635be6e6b3d16132b0c 2024-11-14T00:29:29,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544229108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:29,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/8dd97f24f85041d1a0f9ea76edbae536 is 50, key is test_row_0/C:col10/1731544168278/Put/seqid=0 2024-11-14T00:29:29,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544229121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:29,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544229126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:29,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742245_1421 (size=12001) 2024-11-14T00:29:29,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544229227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:29,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544229227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:29,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:29,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:29,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544229240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:29,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:29,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544229248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:29,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:29,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544229249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:29,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544229445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:29,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544229466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:29,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544229468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:29,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:29,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:29,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/8dd97f24f85041d1a0f9ea76edbae536 2024-11-14T00:29:29,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/2fa7d66fbb0743e28af5eef6aeb18bf0 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/2fa7d66fbb0743e28af5eef6aeb18bf0 2024-11-14T00:29:29,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/2fa7d66fbb0743e28af5eef6aeb18bf0, entries=200, sequenceid=92, filesize=14.0 K 2024-11-14T00:29:29,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/2af7fffbdf1e4635be6e6b3d16132b0c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/2af7fffbdf1e4635be6e6b3d16132b0c 2024-11-14T00:29:29,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/2af7fffbdf1e4635be6e6b3d16132b0c, entries=150, sequenceid=92, filesize=11.7 K 2024-11-14T00:29:29,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/8dd97f24f85041d1a0f9ea76edbae536 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/8dd97f24f85041d1a0f9ea76edbae536 2024-11-14T00:29:29,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/8dd97f24f85041d1a0f9ea76edbae536, entries=150, sequenceid=92, filesize=11.7 K 2024-11-14T00:29:29,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 93f27665e92d07bc413648d25770240d in 694ms, sequenceid=92, compaction requested=true 2024-11-14T00:29:29,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:29,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:29,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:29,656 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:29,656 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:29,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:29,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:29,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:29,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:29,666 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43126 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:29,666 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:29,666 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/A is initiating minor compaction (all files) 2024-11-14T00:29:29,666 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/B is initiating minor compaction (all files) 2024-11-14T00:29:29,666 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/A in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:29,666 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/B in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:29,666 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/f5c0b1799a8642948bff3cb80e498611, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/b4fc5f8b09cb4ed7b5972dad29d7dfe8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/2fa7d66fbb0743e28af5eef6aeb18bf0] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=42.1 K 2024-11-14T00:29:29,666 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/186c49d849b94e6c864295f764248c5b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/16a67c68d3ac4265a51a708399f13e44, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/2af7fffbdf1e4635be6e6b3d16132b0c] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=35.3 K 2024-11-14T00:29:29,672 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c0b1799a8642948bff3cb80e498611, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544167840 2024-11-14T00:29:29,672 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 186c49d849b94e6c864295f764248c5b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544167840 2024-11-14T00:29:29,676 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4fc5f8b09cb4ed7b5972dad29d7dfe8, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731544168038 2024-11-14T00:29:29,680 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 16a67c68d3ac4265a51a708399f13e44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731544168040 2024-11-14T00:29:29,682 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fa7d66fbb0743e28af5eef6aeb18bf0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731544168277 2024-11-14T00:29:29,683 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 2af7fffbdf1e4635be6e6b3d16132b0c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731544168278 2024-11-14T00:29:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T00:29:29,685 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:29,686 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:29:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-14T00:29:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T00:29:29,689 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:29:29,690 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:29:29,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:29:29,700 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#B#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:29,701 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/a81120bff41c479dabaacb7f402fe965 is 50, key is test_row_0/B:col10/1731544168278/Put/seqid=0 2024-11-14T00:29:29,727 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#A#compaction#352 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:29,728 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/126a012beca041a6aef54893ef9a03bb is 50, key is test_row_0/A:col10/1731544168278/Put/seqid=0 2024-11-14T00:29:29,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742246_1422 (size=12104) 2024-11-14T00:29:29,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742247_1423 (size=12104) 2024-11-14T00:29:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:29,783 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:29,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T00:29:29,793 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/126a012beca041a6aef54893ef9a03bb as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/126a012beca041a6aef54893ef9a03bb 2024-11-14T00:29:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T00:29:29,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/cea7e8deddfc494696adc7f1c3bb6603 is 50, key is test_row_0/A:col10/1731544169780/Put/seqid=0 2024-11-14T00:29:29,803 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/A of 93f27665e92d07bc413648d25770240d into 126a012beca041a6aef54893ef9a03bb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:29,803 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:29,803 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/A, priority=13, startTime=1731544169656; duration=0sec 2024-11-14T00:29:29,803 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:29,803 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:A 2024-11-14T00:29:29,803 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:29,804 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:29,804 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/C is initiating minor compaction (all files) 2024-11-14T00:29:29,804 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/C in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:29,804 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/27e1925ef9664e5ab6958d5e82d1d346, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/272f1fdb574f47fc80f78c7deecb88ca, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/8dd97f24f85041d1a0f9ea76edbae536] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=35.3 K 2024-11-14T00:29:29,804 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27e1925ef9664e5ab6958d5e82d1d346, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731544167840 2024-11-14T00:29:29,804 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 272f1fdb574f47fc80f78c7deecb88ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731544168040 2024-11-14T00:29:29,806 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dd97f24f85041d1a0f9ea76edbae536, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731544168278 2024-11-14T00:29:29,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742248_1424 (size=14341) 2024-11-14T00:29:29,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/cea7e8deddfc494696adc7f1c3bb6603 2024-11-14T00:29:29,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T00:29:29,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:29,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:29,845 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#C#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:29,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:29,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:29,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:29,846 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/d7d01cbdbdd74bddbe826236cdd6a056 is 50, key is test_row_0/C:col10/1731544168278/Put/seqid=0 2024-11-14T00:29:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:29,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/3268136d5fa64a83a261e52850dd68bb is 50, key is test_row_0/B:col10/1731544169780/Put/seqid=0 2024-11-14T00:29:29,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544229838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:29,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:29,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544229838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:29,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544229855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:29,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742250_1426 (size=12001) 2024-11-14T00:29:29,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/3268136d5fa64a83a261e52850dd68bb 2024-11-14T00:29:29,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742249_1425 (size=12104) 2024-11-14T00:29:29,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/4e61b526e91147c9863df684455bb356 is 50, key is test_row_0/C:col10/1731544169780/Put/seqid=0 2024-11-14T00:29:29,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742251_1427 (size=12001) 2024-11-14T00:29:29,957 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/d7d01cbdbdd74bddbe826236cdd6a056 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/d7d01cbdbdd74bddbe826236cdd6a056 2024-11-14T00:29:29,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/4e61b526e91147c9863df684455bb356 2024-11-14T00:29:29,965 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/C of 93f27665e92d07bc413648d25770240d into d7d01cbdbdd74bddbe826236cdd6a056(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:29,965 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:29,965 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/C, priority=13, startTime=1731544169656; duration=0sec 2024-11-14T00:29:29,965 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:29,965 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:C 2024-11-14T00:29:29,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544229968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:29,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544229970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:29,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/cea7e8deddfc494696adc7f1c3bb6603 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/cea7e8deddfc494696adc7f1c3bb6603 2024-11-14T00:29:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:29,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/cea7e8deddfc494696adc7f1c3bb6603, entries=200, sequenceid=115, filesize=14.0 K 2024-11-14T00:29:29,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/3268136d5fa64a83a261e52850dd68bb as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/3268136d5fa64a83a261e52850dd68bb 2024-11-14T00:29:30,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T00:29:30,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:30,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:30,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:30,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:30,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:30,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/3268136d5fa64a83a261e52850dd68bb, entries=150, sequenceid=115, filesize=11.7 K 2024-11-14T00:29:30,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/4e61b526e91147c9863df684455bb356 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4e61b526e91147c9863df684455bb356 2024-11-14T00:29:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:30,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:30,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544229979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T00:29:30,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4e61b526e91147c9863df684455bb356, entries=150, sequenceid=115, filesize=11.7 K 2024-11-14T00:29:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:30,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 93f27665e92d07bc413648d25770240d in 233ms, sequenceid=115, compaction requested=false 2024-11-14T00:29:30,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:30,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T00:29:30,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:30,168 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T00:29:30,184 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/a81120bff41c479dabaacb7f402fe965 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/a81120bff41c479dabaacb7f402fe965 2024-11-14T00:29:30,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/a64aa20f2faa41ddb29bcbe8cd6ed7c4 is 50, key is test_row_0/A:col10/1731544169816/Put/seqid=0 2024-11-14T00:29:30,204 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/B of 93f27665e92d07bc413648d25770240d into a81120bff41c479dabaacb7f402fe965(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:30,204 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:30,204 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/B, priority=13, startTime=1731544169656; duration=0sec 2024-11-14T00:29:30,204 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:30,204 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:B 2024-11-14T00:29:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:30,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:30,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742252_1428 (size=12001) 2024-11-14T00:29:30,221 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/a64aa20f2faa41ddb29bcbe8cd6ed7c4 2024-11-14T00:29:30,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/4da4b23ffd19426dbdcfd5037c6bd073 is 50, key is test_row_0/B:col10/1731544169816/Put/seqid=0 2024-11-14T00:29:30,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742253_1429 (size=12001) 2024-11-14T00:29:30,261 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/4da4b23ffd19426dbdcfd5037c6bd073 2024-11-14T00:29:30,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/d47d677bcbd54cc5a19c504d6ccf8917 is 50, key is test_row_0/C:col10/1731544169816/Put/seqid=0 2024-11-14T00:29:30,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742254_1430 (size=12001) 2024-11-14T00:29:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T00:29:30,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544230303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:30,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544230310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:30,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544230305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:30,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544230324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:30,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:30,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544230329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:30,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544230436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544230436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:30,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544230437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:30,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544230447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:30,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:30,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544230455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:30,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:30,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544230655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:30,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:30,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544230656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:30,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544230660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,676 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:30,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544230666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:30,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544230675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:30,720 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/d47d677bcbd54cc5a19c504d6ccf8917 2024-11-14T00:29:30,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/a64aa20f2faa41ddb29bcbe8cd6ed7c4 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/a64aa20f2faa41ddb29bcbe8cd6ed7c4 2024-11-14T00:29:30,735 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/a64aa20f2faa41ddb29bcbe8cd6ed7c4, entries=150, sequenceid=130, filesize=11.7 K 2024-11-14T00:29:30,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/4da4b23ffd19426dbdcfd5037c6bd073 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/4da4b23ffd19426dbdcfd5037c6bd073 2024-11-14T00:29:30,741 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/4da4b23ffd19426dbdcfd5037c6bd073, entries=150, sequenceid=130, filesize=11.7 K 2024-11-14T00:29:30,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/d47d677bcbd54cc5a19c504d6ccf8917 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/d47d677bcbd54cc5a19c504d6ccf8917 2024-11-14T00:29:30,751 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/d47d677bcbd54cc5a19c504d6ccf8917, entries=150, sequenceid=130, filesize=11.7 K 2024-11-14T00:29:30,755 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 93f27665e92d07bc413648d25770240d in 587ms, sequenceid=130, compaction requested=true 2024-11-14T00:29:30,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:30,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:30,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-14T00:29:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-14T00:29:30,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-14T00:29:30,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0780 sec 2024-11-14T00:29:30,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.0860 sec 2024-11-14T00:29:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T00:29:30,826 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:30,826 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:29:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-14T00:29:30,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T00:29:30,831 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:29:30,836 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:29:30,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:29:30,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T00:29:30,988 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:30,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T00:29:30,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:30,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T00:29:30,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:30,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:30,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:30,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:30,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:31,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/efe68c4794ba46a6b16e2329e5b5db95 is 50, key is test_row_0/A:col10/1731544170986/Put/seqid=0 2024-11-14T00:29:31,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742255_1431 (size=16931) 2024-11-14T00:29:31,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/efe68c4794ba46a6b16e2329e5b5db95 2024-11-14T00:29:31,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544231027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544231029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544231029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544231057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544231062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/e27de5ab1d464fd4983329ce9734f036 is 50, key is test_row_0/B:col10/1731544170986/Put/seqid=0 2024-11-14T00:29:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T00:29:31,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742256_1432 (size=12151) 2024-11-14T00:29:31,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T00:29:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:31,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:31,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:31,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544231175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544231175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544231176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544231185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544231185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T00:29:31,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:31,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:31,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:31,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:31,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:31,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544231403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544231406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544231406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544231408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544231408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T00:29:31,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T00:29:31,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:31,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:31,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:31,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:31,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:31,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/e27de5ab1d464fd4983329ce9734f036 2024-11-14T00:29:31,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/9cf228b96462420e87161dc39c6378ed is 50, key is test_row_0/C:col10/1731544170986/Put/seqid=0 2024-11-14T00:29:31,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742257_1433 (size=12151) 2024-11-14T00:29:31,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/9cf228b96462420e87161dc39c6378ed 2024-11-14T00:29:31,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/efe68c4794ba46a6b16e2329e5b5db95 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/efe68c4794ba46a6b16e2329e5b5db95 2024-11-14T00:29:31,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/efe68c4794ba46a6b16e2329e5b5db95, entries=250, sequenceid=156, filesize=16.5 K 2024-11-14T00:29:31,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/e27de5ab1d464fd4983329ce9734f036 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/e27de5ab1d464fd4983329ce9734f036 2024-11-14T00:29:31,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/e27de5ab1d464fd4983329ce9734f036, entries=150, sequenceid=156, filesize=11.9 K 2024-11-14T00:29:31,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/9cf228b96462420e87161dc39c6378ed as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/9cf228b96462420e87161dc39c6378ed 2024-11-14T00:29:31,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/9cf228b96462420e87161dc39c6378ed, entries=150, sequenceid=156, filesize=11.9 K 2024-11-14T00:29:31,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 93f27665e92d07bc413648d25770240d in 663ms, sequenceid=156, compaction requested=true 2024-11-14T00:29:31,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:31,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:31,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:31,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:31,652 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:29:31,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:31,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:31,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:29:31,654 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:29:31,658 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55377 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:29:31,658 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/A is initiating minor compaction (all files) 2024-11-14T00:29:31,658 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/A in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:31,658 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/126a012beca041a6aef54893ef9a03bb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/cea7e8deddfc494696adc7f1c3bb6603, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/a64aa20f2faa41ddb29bcbe8cd6ed7c4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/efe68c4794ba46a6b16e2329e5b5db95] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=54.1 K 2024-11-14T00:29:31,658 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 126a012beca041a6aef54893ef9a03bb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731544168278 2024-11-14T00:29:31,659 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting cea7e8deddfc494696adc7f1c3bb6603, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731544169087 2024-11-14T00:29:31,659 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting a64aa20f2faa41ddb29bcbe8cd6ed7c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731544169816 2024-11-14T00:29:31,660 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting efe68c4794ba46a6b16e2329e5b5db95, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731544170300 2024-11-14T00:29:31,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T00:29:31,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:31,666 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T00:29:31,672 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:29:31,672 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/B is initiating minor compaction (all files) 2024-11-14T00:29:31,672 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/B in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:31,672 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/a81120bff41c479dabaacb7f402fe965, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/3268136d5fa64a83a261e52850dd68bb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/4da4b23ffd19426dbdcfd5037c6bd073, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/e27de5ab1d464fd4983329ce9734f036] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=47.1 K 2024-11-14T00:29:31,676 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting a81120bff41c479dabaacb7f402fe965, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731544168278 2024-11-14T00:29:31,680 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#A#compaction#363 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:31,680 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3268136d5fa64a83a261e52850dd68bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731544169087 2024-11-14T00:29:31,681 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/d913e351257742b29f5504784eb667fb is 50, key is test_row_0/A:col10/1731544170986/Put/seqid=0 2024-11-14T00:29:31,684 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4da4b23ffd19426dbdcfd5037c6bd073, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731544169816 2024-11-14T00:29:31,688 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting e27de5ab1d464fd4983329ce9734f036, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731544170300 2024-11-14T00:29:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/23f765f6b4484f13b80bf48253a0e286 is 50, key is test_row_0/A:col10/1731544171058/Put/seqid=0 2024-11-14T00:29:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:31,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:31,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742259_1435 (size=12151) 2024-11-14T00:29:31,743 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#B#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:31,743 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/0cbeaa10039f484399acf6d1de333745 is 50, key is test_row_0/B:col10/1731544170986/Put/seqid=0 2024-11-14T00:29:31,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742258_1434 (size=12391) 2024-11-14T00:29:31,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742260_1436 (size=12391) 2024-11-14T00:29:31,800 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/d913e351257742b29f5504784eb667fb as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/d913e351257742b29f5504784eb667fb 2024-11-14T00:29:31,809 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 93f27665e92d07bc413648d25770240d/A of 93f27665e92d07bc413648d25770240d into d913e351257742b29f5504784eb667fb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:31,809 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:31,809 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/A, priority=12, startTime=1731544171652; duration=0sec 2024-11-14T00:29:31,809 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:31,809 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:A 2024-11-14T00:29:31,809 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:29:31,810 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:29:31,811 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/C is initiating minor compaction (all files) 2024-11-14T00:29:31,811 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/C in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:31,811 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/d7d01cbdbdd74bddbe826236cdd6a056, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4e61b526e91147c9863df684455bb356, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/d47d677bcbd54cc5a19c504d6ccf8917, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/9cf228b96462420e87161dc39c6378ed] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=47.1 K 2024-11-14T00:29:31,814 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting d7d01cbdbdd74bddbe826236cdd6a056, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731544168278 2024-11-14T00:29:31,815 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e61b526e91147c9863df684455bb356, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731544169087 2024-11-14T00:29:31,824 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting d47d677bcbd54cc5a19c504d6ccf8917, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731544169816 2024-11-14T00:29:31,825 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cf228b96462420e87161dc39c6378ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731544170300 2024-11-14T00:29:31,837 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#C#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:31,837 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/4d08f7a1c2b64b0db60d548ea5973fb3 is 50, key is test_row_0/C:col10/1731544170986/Put/seqid=0 2024-11-14T00:29:31,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742261_1437 (size=12391) 2024-11-14T00:29:31,880 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/4d08f7a1c2b64b0db60d548ea5973fb3 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4d08f7a1c2b64b0db60d548ea5973fb3 2024-11-14T00:29:31,885 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 93f27665e92d07bc413648d25770240d/C of 93f27665e92d07bc413648d25770240d into 4d08f7a1c2b64b0db60d548ea5973fb3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:31,885 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:31,885 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/C, priority=12, startTime=1731544171652; duration=0sec 2024-11-14T00:29:31,885 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:31,885 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:C 2024-11-14T00:29:31,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544231857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544231859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544231862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544231875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:31,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544231876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:31,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T00:29:32,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544232006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544232006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:32,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544232006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544232009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:32,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544232009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:32,140 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/23f765f6b4484f13b80bf48253a0e286 2024-11-14T00:29:32,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/758d7dd144744037a60112bdc36db214 is 50, key is test_row_0/B:col10/1731544171058/Put/seqid=0 2024-11-14T00:29:32,206 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/0cbeaa10039f484399acf6d1de333745 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/0cbeaa10039f484399acf6d1de333745 2024-11-14T00:29:32,213 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 93f27665e92d07bc413648d25770240d/B of 93f27665e92d07bc413648d25770240d into 0cbeaa10039f484399acf6d1de333745(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:32,213 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:32,213 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/B, priority=12, startTime=1731544171652; duration=0sec 2024-11-14T00:29:32,213 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:32,213 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:B 2024-11-14T00:29:32,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544232227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544232227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:32,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:32,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544232228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:32,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:32,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544232235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:32,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544232236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:32,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742262_1438 (size=12151) 2024-11-14T00:29:32,242 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/758d7dd144744037a60112bdc36db214 2024-11-14T00:29:32,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/be0cb009154f463ca29e2a433d5486a6 is 50, key is test_row_0/C:col10/1731544171058/Put/seqid=0 2024-11-14T00:29:32,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742263_1439 (size=12151) 2024-11-14T00:29:32,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:32,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544232547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544232547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:32,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:32,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544232547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:32,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544232547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:32,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544232547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:32,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:32,757 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/be0cb009154f463ca29e2a433d5486a6 2024-11-14T00:29:32,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/23f765f6b4484f13b80bf48253a0e286 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/23f765f6b4484f13b80bf48253a0e286 2024-11-14T00:29:32,845 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/23f765f6b4484f13b80bf48253a0e286, entries=150, sequenceid=167, filesize=11.9 K 2024-11-14T00:29:32,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/758d7dd144744037a60112bdc36db214 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/758d7dd144744037a60112bdc36db214 2024-11-14T00:29:32,889 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/758d7dd144744037a60112bdc36db214, entries=150, sequenceid=167, filesize=11.9 K 2024-11-14T00:29:32,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/be0cb009154f463ca29e2a433d5486a6 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/be0cb009154f463ca29e2a433d5486a6 2024-11-14T00:29:32,918 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/be0cb009154f463ca29e2a433d5486a6, entries=150, sequenceid=167, filesize=11.9 K 2024-11-14T00:29:32,924 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 93f27665e92d07bc413648d25770240d in 1258ms, sequenceid=167, compaction requested=false 2024-11-14T00:29:32,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:32,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:32,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-14T00:29:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-14T00:29:32,931 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-14T00:29:32,931 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0930 sec 2024-11-14T00:29:32,960 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.1140 sec 2024-11-14T00:29:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T00:29:32,997 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:33,000 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:29:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-14T00:29:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T00:29:33,020 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:29:33,021 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:29:33,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:29:33,085 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:33,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T00:29:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:33,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/a912bd8b86604819992dfa0ea7b29c0a is 50, key is test_row_0/A:col10/1731544173081/Put/seqid=0 2024-11-14T00:29:33,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544233093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544233103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544233112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T00:29:33,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544233120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544233126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742264_1440 (size=12151) 2024-11-14T00:29:33,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T00:29:33,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:33,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:33,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:33,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:33,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:33,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:33,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544233236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544233240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544233240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544233248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544233248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T00:29:33,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T00:29:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:33,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:33,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:33,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:33,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:33,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544233455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544233456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544233456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544233466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544233466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T00:29:33,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:33,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:33,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:33,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:33,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:33,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/a912bd8b86604819992dfa0ea7b29c0a 2024-11-14T00:29:33,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/1695b88d7e8048b2a0a74f2a4c47903d is 50, key is test_row_0/B:col10/1731544173081/Put/seqid=0 2024-11-14T00:29:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T00:29:33,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T00:29:33,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:33,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:33,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:33,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:33,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:33,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742265_1441 (size=12151) 2024-11-14T00:29:33,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544233777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544233780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544233786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544233786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544233788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:33,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T00:29:33,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:33,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:33,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:33,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:33,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:34,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T00:29:34,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:34,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:34,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:34,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:34,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:34,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/1695b88d7e8048b2a0a74f2a4c47903d 2024-11-14T00:29:34,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/8ab3705c0cb94a04ab69e4b830acc305 is 50, key is test_row_0/C:col10/1731544173081/Put/seqid=0 2024-11-14T00:29:34,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T00:29:34,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T00:29:34,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:34,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:34,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:34,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:34,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:34,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:34,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742266_1442 (size=12151) 2024-11-14T00:29:34,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544234295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:34,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:34,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:34,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:34,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:34,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544234296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:34,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:34,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544234305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:34,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:34,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:34,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:34,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544234315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:34,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:34,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:34,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:34,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544234320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:34,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T00:29:34,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:34,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:34,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:34,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:34,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:34,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:34,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T00:29:34,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:34,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:34,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:34,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:34,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:34,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:34,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/8ab3705c0cb94a04ab69e4b830acc305 2024-11-14T00:29:34,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/a912bd8b86604819992dfa0ea7b29c0a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/a912bd8b86604819992dfa0ea7b29c0a 2024-11-14T00:29:34,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/a912bd8b86604819992dfa0ea7b29c0a, entries=150, sequenceid=197, filesize=11.9 K 2024-11-14T00:29:34,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T00:29:34,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:34,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:34,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:34,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:34,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:34,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/1695b88d7e8048b2a0a74f2a4c47903d as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1695b88d7e8048b2a0a74f2a4c47903d 2024-11-14T00:29:34,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1695b88d7e8048b2a0a74f2a4c47903d, entries=150, sequenceid=197, filesize=11.9 K 2024-11-14T00:29:34,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/8ab3705c0cb94a04ab69e4b830acc305 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/8ab3705c0cb94a04ab69e4b830acc305 2024-11-14T00:29:34,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/8ab3705c0cb94a04ab69e4b830acc305, entries=150, sequenceid=197, filesize=11.9 K 2024-11-14T00:29:34,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 93f27665e92d07bc413648d25770240d in 1670ms, sequenceid=197, compaction requested=true 2024-11-14T00:29:34,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:34,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:34,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:34,756 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:34,756 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:34,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:34,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:34,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:34,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:34,761 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:34,761 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/A is initiating minor compaction (all files) 2024-11-14T00:29:34,761 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/A in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:34,762 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/d913e351257742b29f5504784eb667fb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/23f765f6b4484f13b80bf48253a0e286, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/a912bd8b86604819992dfa0ea7b29c0a] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=35.8 K 2024-11-14T00:29:34,768 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:34,769 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/B is initiating minor compaction (all files) 2024-11-14T00:29:34,769 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/B in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:34,769 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/0cbeaa10039f484399acf6d1de333745, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/758d7dd144744037a60112bdc36db214, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1695b88d7e8048b2a0a74f2a4c47903d] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=35.8 K 2024-11-14T00:29:34,769 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting d913e351257742b29f5504784eb667fb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731544170300 2024-11-14T00:29:34,770 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cbeaa10039f484399acf6d1de333745, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731544170300 2024-11-14T00:29:34,770 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23f765f6b4484f13b80bf48253a0e286, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731544171025 2024-11-14T00:29:34,770 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 758d7dd144744037a60112bdc36db214, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731544171025 2024-11-14T00:29:34,770 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting a912bd8b86604819992dfa0ea7b29c0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731544171874 2024-11-14T00:29:34,771 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 1695b88d7e8048b2a0a74f2a4c47903d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731544171874 2024-11-14T00:29:34,786 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#A#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:34,787 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/06126d488f234c6e8f7d8f7e68bc330b is 50, key is test_row_0/A:col10/1731544173081/Put/seqid=0 2024-11-14T00:29:34,790 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#B#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:34,790 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/da9df339bf884c95bd5e1f04bc795582 is 50, key is test_row_0/B:col10/1731544173081/Put/seqid=0 2024-11-14T00:29:34,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T00:29:34,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:34,864 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-14T00:29:34,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742268_1444 (size=12493) 2024-11-14T00:29:34,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742267_1443 (size=12493) 2024-11-14T00:29:34,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/823b6175595547f29c7e744142eb8b10 is 50, key is test_row_0/A:col10/1731544173086/Put/seqid=0 2024-11-14T00:29:34,952 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/da9df339bf884c95bd5e1f04bc795582 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/da9df339bf884c95bd5e1f04bc795582 2024-11-14T00:29:34,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742269_1445 (size=9757) 2024-11-14T00:29:34,976 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/B of 93f27665e92d07bc413648d25770240d into da9df339bf884c95bd5e1f04bc795582(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:34,976 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:34,976 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/B, priority=13, startTime=1731544174756; duration=0sec 2024-11-14T00:29:34,976 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:34,977 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:B 2024-11-14T00:29:34,977 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:34,978 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:34,978 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/C is initiating minor compaction (all files) 2024-11-14T00:29:34,978 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/C in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:34,978 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4d08f7a1c2b64b0db60d548ea5973fb3, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/be0cb009154f463ca29e2a433d5486a6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/8ab3705c0cb94a04ab69e4b830acc305] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=35.8 K 2024-11-14T00:29:34,980 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d08f7a1c2b64b0db60d548ea5973fb3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731544170300 2024-11-14T00:29:34,984 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting be0cb009154f463ca29e2a433d5486a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731544171025 2024-11-14T00:29:34,988 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ab3705c0cb94a04ab69e4b830acc305, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731544171874 2024-11-14T00:29:35,039 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#C#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:35,040 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/5d783a3b284e455ba5dfa6a634c29f8b is 50, key is test_row_0/C:col10/1731544173081/Put/seqid=0 2024-11-14T00:29:35,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742270_1446 (size=12493) 2024-11-14T00:29:35,092 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/5d783a3b284e455ba5dfa6a634c29f8b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/5d783a3b284e455ba5dfa6a634c29f8b 2024-11-14T00:29:35,133 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/C of 93f27665e92d07bc413648d25770240d into 5d783a3b284e455ba5dfa6a634c29f8b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:35,133 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:35,133 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/C, priority=13, startTime=1731544174756; duration=0sec 2024-11-14T00:29:35,133 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:35,133 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:C 2024-11-14T00:29:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T00:29:35,330 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/06126d488f234c6e8f7d8f7e68bc330b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/06126d488f234c6e8f7d8f7e68bc330b 2024-11-14T00:29:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:35,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:35,352 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/A of 93f27665e92d07bc413648d25770240d into 06126d488f234c6e8f7d8f7e68bc330b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:35,352 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:35,353 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/A, priority=13, startTime=1731544174756; duration=0sec 2024-11-14T00:29:35,353 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:35,353 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:A 2024-11-14T00:29:35,374 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/823b6175595547f29c7e744142eb8b10 2024-11-14T00:29:35,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/ea4ecfdd382944caa2d2edec7bbaff3a is 50, key is test_row_0/B:col10/1731544173086/Put/seqid=0 2024-11-14T00:29:35,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742271_1447 (size=9757) 2024-11-14T00:29:35,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544235452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:35,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544235461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:35,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:35,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544235461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:35,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544235476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:35,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544235477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:35,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544235585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544235586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:35,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544235588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:35,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:35,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544235605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:35,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:35,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544235606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:35,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:35,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544235807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:35,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544235808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:35,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544235815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:35,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544235816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:35,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:35,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544235828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:35,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:35,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:35,861 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/ea4ecfdd382944caa2d2edec7bbaff3a 2024-11-14T00:29:35,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/41c2989136a84e17bc16e86bae6de5c7 is 50, key is test_row_0/C:col10/1731544173086/Put/seqid=0 2024-11-14T00:29:35,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742272_1448 (size=9757) 2024-11-14T00:29:35,966 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/41c2989136a84e17bc16e86bae6de5c7 2024-11-14T00:29:36,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/823b6175595547f29c7e744142eb8b10 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/823b6175595547f29c7e744142eb8b10 2024-11-14T00:29:36,074 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/823b6175595547f29c7e744142eb8b10, entries=100, sequenceid=206, filesize=9.5 K 2024-11-14T00:29:36,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/ea4ecfdd382944caa2d2edec7bbaff3a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/ea4ecfdd382944caa2d2edec7bbaff3a 2024-11-14T00:29:36,112 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/ea4ecfdd382944caa2d2edec7bbaff3a, entries=100, sequenceid=206, filesize=9.5 K 2024-11-14T00:29:36,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/41c2989136a84e17bc16e86bae6de5c7 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/41c2989136a84e17bc16e86bae6de5c7 2024-11-14T00:29:36,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544236125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:36,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544236128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:36,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544236136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:36,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544236145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:36,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:36,161 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/41c2989136a84e17bc16e86bae6de5c7, entries=100, sequenceid=206, filesize=9.5 K 2024-11-14T00:29:36,165 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for 93f27665e92d07bc413648d25770240d in 1301ms, sequenceid=206, compaction requested=false 2024-11-14T00:29:36,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:36,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:36,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-14T00:29:36,168 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:36,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-14T00:29:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-14T00:29:36,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-14T00:29:36,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1590 sec 2024-11-14T00:29:36,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/eb3afb3d4c14424dbb931ff84e4b719b is 50, key is test_row_0/A:col10/1731544175431/Put/seqid=0 2024-11-14T00:29:36,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 3.1860 sec 2024-11-14T00:29:36,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742273_1449 (size=14541) 2024-11-14T00:29:36,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544236240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:36,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:36,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544236365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:36,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:36,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544236576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:36,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:36,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/eb3afb3d4c14424dbb931ff84e4b719b 2024-11-14T00:29:36,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544236645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:36,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/8f5b225aadfd4959b29e9990ccb3c7d2 is 50, key is test_row_0/B:col10/1731544175431/Put/seqid=0 2024-11-14T00:29:36,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544236656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:36,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:36,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544236665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:36,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544236666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:36,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742274_1450 (size=12151) 2024-11-14T00:29:36,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:36,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544236885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:36,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:36,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:37,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/8f5b225aadfd4959b29e9990ccb3c7d2 2024-11-14T00:29:37,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/3dffbde0d9f64201b8a8eb86d163ef46 is 50, key is test_row_0/C:col10/1731544175431/Put/seqid=0 2024-11-14T00:29:37,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742275_1451 (size=12151) 2024-11-14T00:29:37,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/3dffbde0d9f64201b8a8eb86d163ef46 2024-11-14T00:29:37,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/eb3afb3d4c14424dbb931ff84e4b719b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/eb3afb3d4c14424dbb931ff84e4b719b 2024-11-14T00:29:37,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/eb3afb3d4c14424dbb931ff84e4b719b, entries=200, sequenceid=238, filesize=14.2 K 2024-11-14T00:29:37,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/8f5b225aadfd4959b29e9990ccb3c7d2 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/8f5b225aadfd4959b29e9990ccb3c7d2 2024-11-14T00:29:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T00:29:37,185 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:37,186 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:29:37,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-14T00:29:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T00:29:37,188 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:29:37,190 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:29:37,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:29:37,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/8f5b225aadfd4959b29e9990ccb3c7d2, entries=150, sequenceid=238, filesize=11.9 K 2024-11-14T00:29:37,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/3dffbde0d9f64201b8a8eb86d163ef46 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/3dffbde0d9f64201b8a8eb86d163ef46 2024-11-14T00:29:37,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/3dffbde0d9f64201b8a8eb86d163ef46, entries=150, sequenceid=238, filesize=11.9 K 2024-11-14T00:29:37,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 93f27665e92d07bc413648d25770240d in 1053ms, sequenceid=238, compaction requested=true 2024-11-14T00:29:37,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:37,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:37,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:37,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:37,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:29:37,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:37,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T00:29:37,222 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:37,222 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:37,223 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:37,223 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/C is initiating minor compaction (all files) 2024-11-14T00:29:37,223 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/C in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:37,223 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/5d783a3b284e455ba5dfa6a634c29f8b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/41c2989136a84e17bc16e86bae6de5c7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/3dffbde0d9f64201b8a8eb86d163ef46] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=33.6 K 2024-11-14T00:29:37,223 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36791 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:37,223 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/A is initiating minor compaction (all files) 2024-11-14T00:29:37,223 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/A in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:37,224 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/06126d488f234c6e8f7d8f7e68bc330b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/823b6175595547f29c7e744142eb8b10, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/eb3afb3d4c14424dbb931ff84e4b719b] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=35.9 K 2024-11-14T00:29:37,224 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d783a3b284e455ba5dfa6a634c29f8b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731544171874 2024-11-14T00:29:37,224 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06126d488f234c6e8f7d8f7e68bc330b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731544171874 2024-11-14T00:29:37,224 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 823b6175595547f29c7e744142eb8b10, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731544173086 2024-11-14T00:29:37,224 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 41c2989136a84e17bc16e86bae6de5c7, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731544173086 2024-11-14T00:29:37,225 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb3afb3d4c14424dbb931ff84e4b719b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731544175431 2024-11-14T00:29:37,225 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dffbde0d9f64201b8a8eb86d163ef46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731544175431 2024-11-14T00:29:37,242 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#C#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:37,242 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/c426188b545f421d9bb077563713480b is 50, key is test_row_0/C:col10/1731544175431/Put/seqid=0 2024-11-14T00:29:37,247 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#A#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:37,247 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/4139f465f8144710b582647aa1829938 is 50, key is test_row_0/A:col10/1731544175431/Put/seqid=0 2024-11-14T00:29:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T00:29:37,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742276_1452 (size=12595) 2024-11-14T00:29:37,317 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/c426188b545f421d9bb077563713480b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/c426188b545f421d9bb077563713480b 2024-11-14T00:29:37,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742277_1453 (size=12595) 2024-11-14T00:29:37,330 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/C of 93f27665e92d07bc413648d25770240d into c426188b545f421d9bb077563713480b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:37,330 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:37,330 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/C, priority=13, startTime=1731544177222; duration=0sec 2024-11-14T00:29:37,330 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:37,331 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:C 2024-11-14T00:29:37,331 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:37,334 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/4139f465f8144710b582647aa1829938 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/4139f465f8144710b582647aa1829938 2024-11-14T00:29:37,334 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:37,334 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/B is initiating minor compaction (all files) 2024-11-14T00:29:37,334 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/B in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:37,335 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/da9df339bf884c95bd5e1f04bc795582, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/ea4ecfdd382944caa2d2edec7bbaff3a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/8f5b225aadfd4959b29e9990ccb3c7d2] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=33.6 K 2024-11-14T00:29:37,335 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting da9df339bf884c95bd5e1f04bc795582, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731544171874 2024-11-14T00:29:37,336 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting ea4ecfdd382944caa2d2edec7bbaff3a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731544173086 2024-11-14T00:29:37,336 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f5b225aadfd4959b29e9990ccb3c7d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731544175431 2024-11-14T00:29:37,340 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/A of 93f27665e92d07bc413648d25770240d into 4139f465f8144710b582647aa1829938(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:37,340 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:37,340 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/A, priority=13, startTime=1731544177221; duration=0sec 2024-11-14T00:29:37,340 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:37,340 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:A 2024-11-14T00:29:37,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T00:29:37,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:37,346 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-14T00:29:37,378 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#B#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:37,379 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/9f27cc09586c4c73a106ea49b966c8fe is 50, key is test_row_0/B:col10/1731544175431/Put/seqid=0 2024-11-14T00:29:37,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/db3446552239428ebd9c8e2efc86b325 is 50, key is test_row_0/A:col10/1731544176212/Put/seqid=0 2024-11-14T00:29:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:37,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:37,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742279_1455 (size=12151) 2024-11-14T00:29:37,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742278_1454 (size=12595) 2024-11-14T00:29:37,462 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/db3446552239428ebd9c8e2efc86b325 2024-11-14T00:29:37,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/a25935f0962e4deaa34fb186618de34e is 50, key is test_row_0/B:col10/1731544176212/Put/seqid=0 2024-11-14T00:29:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T00:29:37,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742280_1456 (size=12151) 2024-11-14T00:29:37,548 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/a25935f0962e4deaa34fb186618de34e 2024-11-14T00:29:37,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/2b45f693ed2e49c2b44c058f8c03b1ab is 50, key is test_row_0/C:col10/1731544176212/Put/seqid=0 2024-11-14T00:29:37,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742281_1457 (size=12151) 2024-11-14T00:29:37,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:37,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544237688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:37,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544237689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:37,702 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_1' in 93f27665e92d07bc413648d25770240d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:37,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:37,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:37,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:37,702 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_0' in 93f27665e92d07bc413648d25770240d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:37,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:37,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:37,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:37,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544237696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:37,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544237701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:37,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544237705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:37,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T00:29:37,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544237826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:37,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544237826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:37,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544237832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:37,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:37,872 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/9f27cc09586c4c73a106ea49b966c8fe as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/9f27cc09586c4c73a106ea49b966c8fe 2024-11-14T00:29:37,878 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/B of 93f27665e92d07bc413648d25770240d into 9f27cc09586c4c73a106ea49b966c8fe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:37,878 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:37,878 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/B, priority=13, startTime=1731544177221; duration=0sec 2024-11-14T00:29:37,879 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:37,879 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:B 2024-11-14T00:29:37,999 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/2b45f693ed2e49c2b44c058f8c03b1ab 2024-11-14T00:29:38,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/db3446552239428ebd9c8e2efc86b325 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/db3446552239428ebd9c8e2efc86b325 2024-11-14T00:29:38,040 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/db3446552239428ebd9c8e2efc86b325, entries=150, sequenceid=247, filesize=11.9 K 2024-11-14T00:29:38,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/a25935f0962e4deaa34fb186618de34e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/a25935f0962e4deaa34fb186618de34e 2024-11-14T00:29:38,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544238036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:38,045 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/a25935f0962e4deaa34fb186618de34e, entries=150, sequenceid=247, filesize=11.9 K 2024-11-14T00:29:38,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544238038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:38,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/2b45f693ed2e49c2b44c058f8c03b1ab as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/2b45f693ed2e49c2b44c058f8c03b1ab 2024-11-14T00:29:38,051 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/2b45f693ed2e49c2b44c058f8c03b1ab, entries=150, sequenceid=247, filesize=11.9 K 2024-11-14T00:29:38,052 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 93f27665e92d07bc413648d25770240d in 707ms, sequenceid=247, compaction requested=false 2024-11-14T00:29:38,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:38,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:38,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-14T00:29:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-14T00:29:38,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-14T00:29:38,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 864 msec 2024-11-14T00:29:38,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 870 msec 2024-11-14T00:29:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:38,069 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:38,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-14T00:29:38,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/8351a188c9fe40b29c42f28704cb3b15 is 50, key is test_row_0/A:col10/1731544177689/Put/seqid=0 2024-11-14T00:29:38,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544238095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:38,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:38,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742282_1458 (size=14741) 2024-11-14T00:29:38,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544238219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:38,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T00:29:38,325 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:38,327 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:29:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-14T00:29:38,333 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:29:38,334 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:29:38,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:29:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T00:29:38,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544238356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:38,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544238357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T00:29:38,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544238437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:38,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:38,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T00:29:38,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:38,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:38,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:38,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:38,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:38,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:38,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/8351a188c9fe40b29c42f28704cb3b15 2024-11-14T00:29:38,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/c89894b6e49b424980e64686d0b4575f is 50, key is test_row_0/B:col10/1731544177689/Put/seqid=0 2024-11-14T00:29:38,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742283_1459 (size=12301) 2024-11-14T00:29:38,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T00:29:38,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:38,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:38,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:38,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:38,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T00:29:38,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544238760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:38,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T00:29:38,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:38,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:38,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:38,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:38,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:38,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:38,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544238875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:38,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:38,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544238875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:38,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:38,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/c89894b6e49b424980e64686d0b4575f 2024-11-14T00:29:38,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T00:29:38,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:38,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:38,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:38,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:38,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:38,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T00:29:38,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/383f5f66b581466ab7287dc6b0efe03b is 50, key is test_row_0/C:col10/1731544177689/Put/seqid=0 2024-11-14T00:29:39,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742284_1460 (size=12301) 2024-11-14T00:29:39,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/383f5f66b581466ab7287dc6b0efe03b 2024-11-14T00:29:39,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/8351a188c9fe40b29c42f28704cb3b15 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/8351a188c9fe40b29c42f28704cb3b15 2024-11-14T00:29:39,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/8351a188c9fe40b29c42f28704cb3b15, entries=200, sequenceid=278, filesize=14.4 K 2024-11-14T00:29:39,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/c89894b6e49b424980e64686d0b4575f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/c89894b6e49b424980e64686d0b4575f 2024-11-14T00:29:39,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/c89894b6e49b424980e64686d0b4575f, entries=150, sequenceid=278, filesize=12.0 K 2024-11-14T00:29:39,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/383f5f66b581466ab7287dc6b0efe03b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/383f5f66b581466ab7287dc6b0efe03b 2024-11-14T00:29:39,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T00:29:39,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:39,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:39,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:39,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:39,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:39,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/383f5f66b581466ab7287dc6b0efe03b, entries=150, sequenceid=278, filesize=12.0 K 2024-11-14T00:29:39,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 93f27665e92d07bc413648d25770240d in 1062ms, sequenceid=278, compaction requested=true 2024-11-14T00:29:39,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:39,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:39,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:39,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:39,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:29:39,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:39,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T00:29:39,132 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:39,133 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:39,137 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39487 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:39,137 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/A is initiating minor compaction (all files) 2024-11-14T00:29:39,137 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/A in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:39,137 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/4139f465f8144710b582647aa1829938, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/db3446552239428ebd9c8e2efc86b325, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/8351a188c9fe40b29c42f28704cb3b15] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=38.6 K 2024-11-14T00:29:39,138 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:39,138 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/C is initiating minor compaction (all files) 2024-11-14T00:29:39,138 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/C in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:39,138 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/c426188b545f421d9bb077563713480b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/2b45f693ed2e49c2b44c058f8c03b1ab, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/383f5f66b581466ab7287dc6b0efe03b] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=36.2 K 2024-11-14T00:29:39,138 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4139f465f8144710b582647aa1829938, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731544175431 2024-11-14T00:29:39,138 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c426188b545f421d9bb077563713480b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731544175431 2024-11-14T00:29:39,139 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting db3446552239428ebd9c8e2efc86b325, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731544176183 2024-11-14T00:29:39,139 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b45f693ed2e49c2b44c058f8c03b1ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731544176183 2024-11-14T00:29:39,139 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8351a188c9fe40b29c42f28704cb3b15, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731544177666 2024-11-14T00:29:39,140 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 383f5f66b581466ab7287dc6b0efe03b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731544177689 2024-11-14T00:29:39,165 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#C#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:39,166 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/25cb9a88dd6b4646813856e3b143133e is 50, key is test_row_0/C:col10/1731544177689/Put/seqid=0 2024-11-14T00:29:39,170 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#A#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:39,171 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/ae34fbecef72466a93a89a6a42273db5 is 50, key is test_row_0/A:col10/1731544177689/Put/seqid=0 2024-11-14T00:29:39,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742285_1461 (size=12847) 2024-11-14T00:29:39,224 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/25cb9a88dd6b4646813856e3b143133e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/25cb9a88dd6b4646813856e3b143133e 2024-11-14T00:29:39,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742286_1462 (size=12847) 2024-11-14T00:29:39,235 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/C of 93f27665e92d07bc413648d25770240d into 25cb9a88dd6b4646813856e3b143133e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:39,235 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:39,235 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/C, priority=13, startTime=1731544179131; duration=0sec 2024-11-14T00:29:39,235 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:39,235 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:C 2024-11-14T00:29:39,235 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:39,242 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:39,242 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/B is initiating minor compaction (all files) 2024-11-14T00:29:39,243 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/B in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:39,243 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/9f27cc09586c4c73a106ea49b966c8fe, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/a25935f0962e4deaa34fb186618de34e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/c89894b6e49b424980e64686d0b4575f] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=36.2 K 2024-11-14T00:29:39,243 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f27cc09586c4c73a106ea49b966c8fe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731544175431 2024-11-14T00:29:39,245 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting a25935f0962e4deaa34fb186618de34e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731544176183 2024-11-14T00:29:39,245 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c89894b6e49b424980e64686d0b4575f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731544177689 2024-11-14T00:29:39,246 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/ae34fbecef72466a93a89a6a42273db5 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/ae34fbecef72466a93a89a6a42273db5 2024-11-14T00:29:39,253 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/A of 93f27665e92d07bc413648d25770240d into ae34fbecef72466a93a89a6a42273db5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:39,253 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:39,253 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/A, priority=13, startTime=1731544179131; duration=0sec 2024-11-14T00:29:39,254 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:39,254 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:A 2024-11-14T00:29:39,261 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#B#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:39,261 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/879d9ec3074440ab979d00c2eec6193f is 50, key is test_row_0/B:col10/1731544177689/Put/seqid=0 2024-11-14T00:29:39,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T00:29:39,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:39,270 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-14T00:29:39,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742287_1463 (size=12847) 2024-11-14T00:29:39,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/7a418677706d454b8d81bb3f4a9ff84e is 50, key is test_row_0/A:col10/1731544178093/Put/seqid=0 2024-11-14T00:29:39,296 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/879d9ec3074440ab979d00c2eec6193f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/879d9ec3074440ab979d00c2eec6193f 2024-11-14T00:29:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:39,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:39,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742288_1464 (size=9857) 2024-11-14T00:29:39,327 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/B of 93f27665e92d07bc413648d25770240d into 879d9ec3074440ab979d00c2eec6193f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:39,327 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:39,327 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/B, priority=13, startTime=1731544179131; duration=0sec 2024-11-14T00:29:39,328 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:39,328 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:B 2024-11-14T00:29:39,328 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/7a418677706d454b8d81bb3f4a9ff84e 2024-11-14T00:29:39,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/3ad8a42f71ff438ebf86139d247779ef is 50, key is test_row_0/B:col10/1731544178093/Put/seqid=0 2024-11-14T00:29:39,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742289_1465 (size=9857) 2024-11-14T00:29:39,377 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/3ad8a42f71ff438ebf86139d247779ef 2024-11-14T00:29:39,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/7f75dde3e052404fbcd2bfd60ce4b713 is 50, key is test_row_0/C:col10/1731544178093/Put/seqid=0 2024-11-14T00:29:39,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742290_1466 (size=9857) 2024-11-14T00:29:39,400 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/7f75dde3e052404fbcd2bfd60ce4b713 2024-11-14T00:29:39,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/7a418677706d454b8d81bb3f4a9ff84e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/7a418677706d454b8d81bb3f4a9ff84e 2024-11-14T00:29:39,419 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/7a418677706d454b8d81bb3f4a9ff84e, entries=100, sequenceid=286, filesize=9.6 K 2024-11-14T00:29:39,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/3ad8a42f71ff438ebf86139d247779ef as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/3ad8a42f71ff438ebf86139d247779ef 2024-11-14T00:29:39,426 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/3ad8a42f71ff438ebf86139d247779ef, entries=100, sequenceid=286, filesize=9.6 K 2024-11-14T00:29:39,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/7f75dde3e052404fbcd2bfd60ce4b713 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7f75dde3e052404fbcd2bfd60ce4b713 2024-11-14T00:29:39,442 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7f75dde3e052404fbcd2bfd60ce4b713, entries=100, sequenceid=286, filesize=9.6 K 2024-11-14T00:29:39,443 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=140.89 KB/144270 for 93f27665e92d07bc413648d25770240d in 174ms, sequenceid=286, compaction requested=false 2024-11-14T00:29:39,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:39,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:39,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-14T00:29:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-14T00:29:39,451 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:39,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T00:29:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:39,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-14T00:29:39,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1110 sec 2024-11-14T00:29:39,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/24608c08eaa041dbad3accdcee8ce122 is 50, key is test_row_0/A:col10/1731544179438/Put/seqid=0 2024-11-14T00:29:39,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.1450 sec 2024-11-14T00:29:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T00:29:39,485 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:39,486 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:29:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-14T00:29:39,488 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:29:39,489 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:29:39,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:29:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T00:29:39,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742291_1467 (size=14741) 2024-11-14T00:29:39,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544239551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:39,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T00:29:39,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T00:29:39,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:39,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:39,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:39,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:39,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:39,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:39,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544239665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:39,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544239729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:39,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544239730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:39,737 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_1' in 93f27665e92d07bc413648d25770240d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:39,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:39,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:39,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:39,738 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_0' in 93f27665e92d07bc413648d25770240d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:39,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T00:29:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:39,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:39,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T00:29:39,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544239885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:39,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:39,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/24608c08eaa041dbad3accdcee8ce122 2024-11-14T00:29:39,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544239895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:39,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544239895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:39,899 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_2' in 93f27665e92d07bc413648d25770240d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:39,900 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_1' in 93f27665e92d07bc413648d25770240d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:39,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/1f10b0643b0f41319972f6c38459e522 is 50, key is test_row_0/B:col10/1731544179438/Put/seqid=0 2024-11-14T00:29:39,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742292_1468 (size=12301) 2024-11-14T00:29:39,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/1f10b0643b0f41319972f6c38459e522 2024-11-14T00:29:39,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/b97ac737643442d3a45694dc10a1b178 is 50, key is test_row_0/C:col10/1731544179438/Put/seqid=0 2024-11-14T00:29:39,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742293_1469 (size=12301) 2024-11-14T00:29:39,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T00:29:39,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:39,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:39,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:39,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:39,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:40,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T00:29:40,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:40,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:40,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:40,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:40,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T00:29:40,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544240205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:40,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T00:29:40,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:40,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:40,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:40,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:40,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:40,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:40,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/b97ac737643442d3a45694dc10a1b178 2024-11-14T00:29:40,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/24608c08eaa041dbad3accdcee8ce122 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/24608c08eaa041dbad3accdcee8ce122 2024-11-14T00:29:40,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/24608c08eaa041dbad3accdcee8ce122, entries=200, sequenceid=312, filesize=14.4 K 2024-11-14T00:29:40,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/1f10b0643b0f41319972f6c38459e522 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1f10b0643b0f41319972f6c38459e522 2024-11-14T00:29:40,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1f10b0643b0f41319972f6c38459e522, entries=150, sequenceid=312, filesize=12.0 K 2024-11-14T00:29:40,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/b97ac737643442d3a45694dc10a1b178 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/b97ac737643442d3a45694dc10a1b178 2024-11-14T00:29:40,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/b97ac737643442d3a45694dc10a1b178, entries=150, sequenceid=312, filesize=12.0 K 2024-11-14T00:29:40,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 93f27665e92d07bc413648d25770240d in 878ms, sequenceid=312, compaction requested=true 2024-11-14T00:29:40,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:40,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:40,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:40,329 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:40,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:40,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:40,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:40,329 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:40,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:40,330 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37445 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:40,330 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:40,330 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/A is initiating minor compaction (all files) 2024-11-14T00:29:40,330 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/B is initiating minor compaction (all files) 2024-11-14T00:29:40,330 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/B in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:40,330 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/A in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:40,330 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/879d9ec3074440ab979d00c2eec6193f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/3ad8a42f71ff438ebf86139d247779ef, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1f10b0643b0f41319972f6c38459e522] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=34.2 K 2024-11-14T00:29:40,330 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/ae34fbecef72466a93a89a6a42273db5, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/7a418677706d454b8d81bb3f4a9ff84e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/24608c08eaa041dbad3accdcee8ce122] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=36.6 K 2024-11-14T00:29:40,331 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 879d9ec3074440ab979d00c2eec6193f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731544177689 2024-11-14T00:29:40,331 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae34fbecef72466a93a89a6a42273db5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731544177689 2024-11-14T00:29:40,331 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ad8a42f71ff438ebf86139d247779ef, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731544178088 2024-11-14T00:29:40,331 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a418677706d454b8d81bb3f4a9ff84e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731544178088 2024-11-14T00:29:40,331 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f10b0643b0f41319972f6c38459e522, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731544179419 2024-11-14T00:29:40,331 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24608c08eaa041dbad3accdcee8ce122, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731544179395 2024-11-14T00:29:40,338 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#B#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:40,338 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#A#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:40,338 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/1216b70b992940c988c045f8b0e22848 is 50, key is test_row_0/A:col10/1731544179438/Put/seqid=0 2024-11-14T00:29:40,338 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/73b028b37fb44771a30fa0c440ca9cae is 50, key is test_row_0/B:col10/1731544179438/Put/seqid=0 2024-11-14T00:29:40,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742294_1470 (size=12949) 2024-11-14T00:29:40,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742295_1471 (size=12949) 2024-11-14T00:29:40,387 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/73b028b37fb44771a30fa0c440ca9cae as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/73b028b37fb44771a30fa0c440ca9cae 2024-11-14T00:29:40,400 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/B of 93f27665e92d07bc413648d25770240d into 73b028b37fb44771a30fa0c440ca9cae(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:40,400 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:40,401 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/B, priority=13, startTime=1731544180329; duration=0sec 2024-11-14T00:29:40,401 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:40,401 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:B 2024-11-14T00:29:40,401 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:40,402 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:40,402 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/C is initiating minor compaction (all files) 2024-11-14T00:29:40,402 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/C in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:40,402 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/25cb9a88dd6b4646813856e3b143133e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7f75dde3e052404fbcd2bfd60ce4b713, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/b97ac737643442d3a45694dc10a1b178] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=34.2 K 2024-11-14T00:29:40,403 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 25cb9a88dd6b4646813856e3b143133e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731544177689 2024-11-14T00:29:40,403 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f75dde3e052404fbcd2bfd60ce4b713, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731544178088 2024-11-14T00:29:40,403 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting b97ac737643442d3a45694dc10a1b178, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731544179419 2024-11-14T00:29:40,411 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#C#compaction#401 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:40,412 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/7f28540e95ac4a08ae2d4824ae098647 is 50, key is test_row_0/C:col10/1731544179438/Put/seqid=0 2024-11-14T00:29:40,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T00:29:40,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:40,412 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T00:29:40,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/bc761337c2fd4f7581b37455806c1328 is 50, key is test_row_0/A:col10/1731544179549/Put/seqid=0 2024-11-14T00:29:40,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742296_1472 (size=12949) 2024-11-14T00:29:40,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742297_1473 (size=12301) 2024-11-14T00:29:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T00:29:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:40,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:40,789 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/1216b70b992940c988c045f8b0e22848 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/1216b70b992940c988c045f8b0e22848 2024-11-14T00:29:40,793 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/A of 93f27665e92d07bc413648d25770240d into 1216b70b992940c988c045f8b0e22848(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:40,794 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:40,794 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/A, priority=13, startTime=1731544180329; duration=0sec 2024-11-14T00:29:40,794 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:40,794 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:A 2024-11-14T00:29:40,846 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/bc761337c2fd4f7581b37455806c1328 2024-11-14T00:29:40,848 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/7f28540e95ac4a08ae2d4824ae098647 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7f28540e95ac4a08ae2d4824ae098647 2024-11-14T00:29:40,855 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/C of 93f27665e92d07bc413648d25770240d into 7f28540e95ac4a08ae2d4824ae098647(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:40,855 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:40,855 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/C, priority=13, startTime=1731544180329; duration=0sec 2024-11-14T00:29:40,855 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:40,855 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:C 2024-11-14T00:29:40,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/7a0798d4c3f44b818af061f319b1e612 is 50, key is test_row_0/B:col10/1731544179549/Put/seqid=0 2024-11-14T00:29:40,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742298_1474 (size=12301) 2024-11-14T00:29:40,899 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/7a0798d4c3f44b818af061f319b1e612 2024-11-14T00:29:40,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/2132940e1a7a45e9b146c677001f3628 is 50, key is test_row_0/C:col10/1731544179549/Put/seqid=0 2024-11-14T00:29:40,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742299_1475 (size=12301) 2024-11-14T00:29:40,926 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/2132940e1a7a45e9b146c677001f3628 2024-11-14T00:29:40,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544240918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:40,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/bc761337c2fd4f7581b37455806c1328 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/bc761337c2fd4f7581b37455806c1328 2024-11-14T00:29:40,937 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/bc761337c2fd4f7581b37455806c1328, entries=150, sequenceid=324, filesize=12.0 K 2024-11-14T00:29:40,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/7a0798d4c3f44b818af061f319b1e612 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/7a0798d4c3f44b818af061f319b1e612 2024-11-14T00:29:40,942 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/7a0798d4c3f44b818af061f319b1e612, entries=150, sequenceid=324, filesize=12.0 K 2024-11-14T00:29:40,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/2132940e1a7a45e9b146c677001f3628 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/2132940e1a7a45e9b146c677001f3628 2024-11-14T00:29:40,947 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/2132940e1a7a45e9b146c677001f3628, entries=150, sequenceid=324, filesize=12.0 K 2024-11-14T00:29:40,948 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 93f27665e92d07bc413648d25770240d in 536ms, sequenceid=324, compaction requested=false 2024-11-14T00:29:40,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:40,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:40,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-14T00:29:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-14T00:29:40,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-14T00:29:40,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4610 sec 2024-11-14T00:29:40,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.4650 sec 2024-11-14T00:29:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:41,040 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:41,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T00:29:41,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/2a5d002ae6784dada9dbb8d4c6391bb6 is 50, key is test_row_0/A:col10/1731544181038/Put/seqid=0 2024-11-14T00:29:41,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742300_1476 (size=14741) 2024-11-14T00:29:41,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/2a5d002ae6784dada9dbb8d4c6391bb6 2024-11-14T00:29:41,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/7c54685fced1440a905b06f7b11a8b11 is 50, key is test_row_0/B:col10/1731544181038/Put/seqid=0 2024-11-14T00:29:41,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742301_1477 (size=12301) 2024-11-14T00:29:41,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544241104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:41,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:41,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:41,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:41,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544241215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:41,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:41,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544241426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:41,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:41,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/7c54685fced1440a905b06f7b11a8b11 2024-11-14T00:29:41,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/7e1c111dc2264afd957b08a0fa85cc71 is 50, key is test_row_0/C:col10/1731544181038/Put/seqid=0 2024-11-14T00:29:41,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742302_1478 (size=12301) 2024-11-14T00:29:41,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/7e1c111dc2264afd957b08a0fa85cc71 2024-11-14T00:29:41,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/2a5d002ae6784dada9dbb8d4c6391bb6 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/2a5d002ae6784dada9dbb8d4c6391bb6 2024-11-14T00:29:41,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/2a5d002ae6784dada9dbb8d4c6391bb6, entries=200, sequenceid=352, filesize=14.4 K 2024-11-14T00:29:41,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/7c54685fced1440a905b06f7b11a8b11 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/7c54685fced1440a905b06f7b11a8b11 2024-11-14T00:29:41,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/7c54685fced1440a905b06f7b11a8b11, entries=150, sequenceid=352, filesize=12.0 K 2024-11-14T00:29:41,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/7e1c111dc2264afd957b08a0fa85cc71 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7e1c111dc2264afd957b08a0fa85cc71 2024-11-14T00:29:41,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7e1c111dc2264afd957b08a0fa85cc71, entries=150, sequenceid=352, filesize=12.0 K 2024-11-14T00:29:41,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 93f27665e92d07bc413648d25770240d in 523ms, sequenceid=352, compaction requested=true 2024-11-14T00:29:41,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:41,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:41,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:41,563 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:41,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:41,564 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:41,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:41,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:41,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:41,571 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:41,571 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/B is initiating minor compaction (all files) 2024-11-14T00:29:41,571 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/B in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:41,571 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/73b028b37fb44771a30fa0c440ca9cae, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/7a0798d4c3f44b818af061f319b1e612, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/7c54685fced1440a905b06f7b11a8b11] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=36.7 K 2024-11-14T00:29:41,572 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:41,572 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/A is initiating minor compaction (all files) 2024-11-14T00:29:41,572 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/A in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:41,572 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/1216b70b992940c988c045f8b0e22848, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/bc761337c2fd4f7581b37455806c1328, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/2a5d002ae6784dada9dbb8d4c6391bb6] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=39.1 K 2024-11-14T00:29:41,572 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 73b028b37fb44771a30fa0c440ca9cae, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731544179419 2024-11-14T00:29:41,573 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1216b70b992940c988c045f8b0e22848, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731544179419 2024-11-14T00:29:41,573 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a0798d4c3f44b818af061f319b1e612, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731544179511 2024-11-14T00:29:41,573 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc761337c2fd4f7581b37455806c1328, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731544179511 2024-11-14T00:29:41,573 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c54685fced1440a905b06f7b11a8b11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731544180862 2024-11-14T00:29:41,573 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a5d002ae6784dada9dbb8d4c6391bb6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731544180862 2024-11-14T00:29:41,590 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:41,591 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/b009d4154f854b3c80a13eb6e0185610 is 50, key is test_row_0/B:col10/1731544181038/Put/seqid=0 2024-11-14T00:29:41,598 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#A#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:41,599 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/aeebae8d7a664568b251694bee2782d7 is 50, key is test_row_0/A:col10/1731544181038/Put/seqid=0 2024-11-14T00:29:41,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742303_1479 (size=13051) 2024-11-14T00:29:41,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742304_1480 (size=13051) 2024-11-14T00:29:41,614 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/aeebae8d7a664568b251694bee2782d7 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/aeebae8d7a664568b251694bee2782d7 2024-11-14T00:29:41,620 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/A of 93f27665e92d07bc413648d25770240d into aeebae8d7a664568b251694bee2782d7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:41,620 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:41,620 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/A, priority=13, startTime=1731544181563; duration=0sec 2024-11-14T00:29:41,620 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:41,620 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:A 2024-11-14T00:29:41,621 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:41,621 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:41,621 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/C is initiating minor compaction (all files) 2024-11-14T00:29:41,621 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/C in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:41,621 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7f28540e95ac4a08ae2d4824ae098647, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/2132940e1a7a45e9b146c677001f3628, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7e1c111dc2264afd957b08a0fa85cc71] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=36.7 K 2024-11-14T00:29:41,622 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f28540e95ac4a08ae2d4824ae098647, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731544179419 2024-11-14T00:29:41,622 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2132940e1a7a45e9b146c677001f3628, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731544179511 2024-11-14T00:29:41,622 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e1c111dc2264afd957b08a0fa85cc71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731544180862 2024-11-14T00:29:41,635 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#C#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:41,635 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/eadc00944d5943e592fe154b2744fc8a is 50, key is test_row_0/C:col10/1731544181038/Put/seqid=0 2024-11-14T00:29:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T00:29:41,636 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:41,636 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:29:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-14T00:29:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T00:29:41,638 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:29:41,639 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:29:41,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:29:41,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742305_1481 (size=13051) 2024-11-14T00:29:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:41,739 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:41,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T00:29:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T00:29:41,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/d6f96fbf78c44bc08213887a7e343f89 is 50, key is test_row_0/A:col10/1731544181738/Put/seqid=0 2024-11-14T00:29:41,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742306_1482 (size=14741) 2024-11-14T00:29:41,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T00:29:41,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:41,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:41,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:41,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:41,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:41,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T00:29:41,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:41,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:41,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:41,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:41,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T00:29:41,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544241958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:41,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544241959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:41,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:41,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544241971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:42,033 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/b009d4154f854b3c80a13eb6e0185610 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/b009d4154f854b3c80a13eb6e0185610 2024-11-14T00:29:42,044 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/B of 93f27665e92d07bc413648d25770240d into b009d4154f854b3c80a13eb6e0185610(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:42,044 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:42,044 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/B, priority=13, startTime=1731544181563; duration=0sec 2024-11-14T00:29:42,044 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:42,044 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:B 2024-11-14T00:29:42,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:42,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544242076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:42,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544242076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,087 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:42,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544242085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:42,094 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/eadc00944d5943e592fe154b2744fc8a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/eadc00944d5943e592fe154b2744fc8a 2024-11-14T00:29:42,099 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/C of 93f27665e92d07bc413648d25770240d into eadc00944d5943e592fe154b2744fc8a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:42,099 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:42,099 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/C, priority=13, startTime=1731544181564; duration=0sec 2024-11-14T00:29:42,099 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:42,099 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:C 2024-11-14T00:29:42,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T00:29:42,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:42,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:42,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:42,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/d6f96fbf78c44bc08213887a7e343f89 2024-11-14T00:29:42,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/1fb199bbe67c42f6ba0330d1a437e494 is 50, key is test_row_0/B:col10/1731544181738/Put/seqid=0 2024-11-14T00:29:42,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742307_1483 (size=12301) 2024-11-14T00:29:42,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T00:29:42,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:42,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:42,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:42,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T00:29:42,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:42,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544242295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:42,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:42,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544242295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:42,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544242295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:42,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T00:29:42,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:42,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:42,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:42,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T00:29:42,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:42,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:42,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:42,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:42,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544242605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544242605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:42,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:42,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544242605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:42,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:42,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:42,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:42,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/1fb199bbe67c42f6ba0330d1a437e494 2024-11-14T00:29:42,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/46bfa3a17e8a476284435ff6af074c2d is 50, key is test_row_0/C:col10/1731544181738/Put/seqid=0 2024-11-14T00:29:42,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742308_1484 (size=12301) 2024-11-14T00:29:42,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T00:29:42,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:42,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:42,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:42,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T00:29:42,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T00:29:42,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:42,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:42,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:42,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:43,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T00:29:43,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:43,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:43,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:43,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:43,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:43,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/46bfa3a17e8a476284435ff6af074c2d 2024-11-14T00:29:43,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/d6f96fbf78c44bc08213887a7e343f89 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/d6f96fbf78c44bc08213887a7e343f89 2024-11-14T00:29:43,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/d6f96fbf78c44bc08213887a7e343f89, entries=200, sequenceid=364, filesize=14.4 K 2024-11-14T00:29:43,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/1fb199bbe67c42f6ba0330d1a437e494 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1fb199bbe67c42f6ba0330d1a437e494 2024-11-14T00:29:43,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1fb199bbe67c42f6ba0330d1a437e494, entries=150, sequenceid=364, filesize=12.0 K 2024-11-14T00:29:43,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/46bfa3a17e8a476284435ff6af074c2d as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/46bfa3a17e8a476284435ff6af074c2d 2024-11-14T00:29:43,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/46bfa3a17e8a476284435ff6af074c2d, entries=150, sequenceid=364, filesize=12.0 K 2024-11-14T00:29:43,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 93f27665e92d07bc413648d25770240d in 1359ms, sequenceid=364, compaction requested=false 2024-11-14T00:29:43,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:43,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:43,133 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:43,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T00:29:43,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/fa803008ae3a4737a8f712807681bcaf is 50, key is test_row_0/A:col10/1731544183131/Put/seqid=0 2024-11-14T00:29:43,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742309_1485 (size=17181) 2024-11-14T00:29:43,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/fa803008ae3a4737a8f712807681bcaf 2024-11-14T00:29:43,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/6107ba4f6f8f425bb52b28db98d52071 is 50, key is test_row_0/B:col10/1731544183131/Put/seqid=0 2024-11-14T00:29:43,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:43,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544243149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:43,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:43,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742310_1486 (size=12301) 2024-11-14T00:29:43,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/6107ba4f6f8f425bb52b28db98d52071 2024-11-14T00:29:43,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/9c357a70ab404f1e95d0bf27589f7f10 is 50, key is test_row_0/C:col10/1731544183131/Put/seqid=0 2024-11-14T00:29:43,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544243165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:43,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544243165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:43,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742311_1487 (size=12301) 2024-11-14T00:29:43,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T00:29:43,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:43,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:43,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:43,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:43,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:43,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:43,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544243275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:43,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:43,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544243289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:43,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:43,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544243289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:43,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T00:29:43,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:43,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:43,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:43,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:43,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:43,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:43,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544243485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:43,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T00:29:43,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:43,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:43,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:43,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:43,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:43,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:43,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:43,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544243495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:43,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:43,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544243505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:43,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/9c357a70ab404f1e95d0bf27589f7f10 2024-11-14T00:29:43,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/fa803008ae3a4737a8f712807681bcaf as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/fa803008ae3a4737a8f712807681bcaf 2024-11-14T00:29:43,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/fa803008ae3a4737a8f712807681bcaf, entries=250, sequenceid=393, filesize=16.8 K 2024-11-14T00:29:43,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/6107ba4f6f8f425bb52b28db98d52071 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/6107ba4f6f8f425bb52b28db98d52071 2024-11-14T00:29:43,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/6107ba4f6f8f425bb52b28db98d52071, entries=150, sequenceid=393, filesize=12.0 K 2024-11-14T00:29:43,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/9c357a70ab404f1e95d0bf27589f7f10 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/9c357a70ab404f1e95d0bf27589f7f10 2024-11-14T00:29:43,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/9c357a70ab404f1e95d0bf27589f7f10, entries=150, sequenceid=393, filesize=12.0 K 2024-11-14T00:29:43,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 93f27665e92d07bc413648d25770240d in 484ms, sequenceid=393, compaction requested=true 2024-11-14T00:29:43,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:43,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:43,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:43,617 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:43,617 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:43,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:43,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:43,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:43,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:43,619 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:43,619 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/B is initiating minor compaction (all files) 2024-11-14T00:29:43,619 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/B in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:43,619 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44973 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:43,619 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/b009d4154f854b3c80a13eb6e0185610, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1fb199bbe67c42f6ba0330d1a437e494, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/6107ba4f6f8f425bb52b28db98d52071] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=36.8 K 2024-11-14T00:29:43,619 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/A is initiating minor compaction (all files) 2024-11-14T00:29:43,619 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/A in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:43,620 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/aeebae8d7a664568b251694bee2782d7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/d6f96fbf78c44bc08213887a7e343f89, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/fa803008ae3a4737a8f712807681bcaf] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=43.9 K 2024-11-14T00:29:43,620 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting b009d4154f854b3c80a13eb6e0185610, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731544180862 2024-11-14T00:29:43,620 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeebae8d7a664568b251694bee2782d7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731544180862 2024-11-14T00:29:43,622 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6f96fbf78c44bc08213887a7e343f89, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731544181053 2024-11-14T00:29:43,623 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fb199bbe67c42f6ba0330d1a437e494, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731544181083 2024-11-14T00:29:43,623 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa803008ae3a4737a8f712807681bcaf, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731544181948 2024-11-14T00:29:43,623 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 6107ba4f6f8f425bb52b28db98d52071, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731544181956 2024-11-14T00:29:43,635 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#B#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:43,636 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/1943a902ad3f4088b350834c2104a337 is 50, key is test_row_0/B:col10/1731544183131/Put/seqid=0 2024-11-14T00:29:43,650 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#A#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:43,650 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/d77c6830c48b40ef914e9ece92133a1b is 50, key is test_row_0/A:col10/1731544183131/Put/seqid=0 2024-11-14T00:29:43,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T00:29:43,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:43,651 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T00:29:43,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/268395cfc62d4e5b9197f056bb5b16b7 is 50, key is test_row_0/A:col10/1731544183158/Put/seqid=0 2024-11-14T00:29:43,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742312_1488 (size=13153) 2024-11-14T00:29:43,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742313_1489 (size=13153) 2024-11-14T00:29:43,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742314_1490 (size=9857) 2024-11-14T00:29:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:43,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. as already flushing 2024-11-14T00:29:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T00:29:43,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544243906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:43,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:43,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544243906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:43,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:43,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544243911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:43,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544243913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:43,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:43,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:43,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544243907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:43,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:44,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544244036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:44,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544244036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:44,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544244048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:44,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544244049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544244049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:44,108 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/1943a902ad3f4088b350834c2104a337 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1943a902ad3f4088b350834c2104a337 2024-11-14T00:29:44,112 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/B of 93f27665e92d07bc413648d25770240d into 1943a902ad3f4088b350834c2104a337(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:44,112 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:44,112 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/B, priority=13, startTime=1731544183617; duration=0sec 2024-11-14T00:29:44,113 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:44,113 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:B 2024-11-14T00:29:44,113 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:44,114 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:44,114 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/C is initiating minor compaction (all files) 2024-11-14T00:29:44,114 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/C in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:44,114 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/eadc00944d5943e592fe154b2744fc8a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/46bfa3a17e8a476284435ff6af074c2d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/9c357a70ab404f1e95d0bf27589f7f10] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=36.8 K 2024-11-14T00:29:44,114 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting eadc00944d5943e592fe154b2744fc8a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731544180862 2024-11-14T00:29:44,114 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 46bfa3a17e8a476284435ff6af074c2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731544181083 2024-11-14T00:29:44,115 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c357a70ab404f1e95d0bf27589f7f10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731544181956 2024-11-14T00:29:44,126 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#C#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:44,127 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/7d548ec02b594b6c8d9f06f3c41289c8 is 50, key is test_row_0/C:col10/1731544183131/Put/seqid=0 2024-11-14T00:29:44,137 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/d77c6830c48b40ef914e9ece92133a1b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/d77c6830c48b40ef914e9ece92133a1b 2024-11-14T00:29:44,140 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/268395cfc62d4e5b9197f056bb5b16b7 2024-11-14T00:29:44,143 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/A of 93f27665e92d07bc413648d25770240d into d77c6830c48b40ef914e9ece92133a1b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:44,143 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:44,143 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/A, priority=13, startTime=1731544183617; duration=0sec 2024-11-14T00:29:44,143 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:44,143 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:A 2024-11-14T00:29:44,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/533acdddb20d4b3da650d1949312c8e6 is 50, key is test_row_0/B:col10/1731544183158/Put/seqid=0 2024-11-14T00:29:44,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742315_1491 (size=13153) 2024-11-14T00:29:44,169 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/7d548ec02b594b6c8d9f06f3c41289c8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7d548ec02b594b6c8d9f06f3c41289c8 2024-11-14T00:29:44,174 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/C of 93f27665e92d07bc413648d25770240d into 7d548ec02b594b6c8d9f06f3c41289c8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:44,174 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:44,174 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/C, priority=13, startTime=1731544183618; duration=0sec 2024-11-14T00:29:44,174 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:44,174 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:C 2024-11-14T00:29:44,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742316_1492 (size=9857) 2024-11-14T00:29:44,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544244265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:44,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544244265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:44,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544244265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544244275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:44,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544244275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,279 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:44,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544244586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:44,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544244586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544244586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:44,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544244587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:44,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544244587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:44,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:44,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:44,610 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/533acdddb20d4b3da650d1949312c8e6 2024-11-14T00:29:44,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/049bcc2d13b24b748df83b2192b2965b is 50, key is test_row_0/C:col10/1731544183158/Put/seqid=0 2024-11-14T00:29:44,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742317_1493 (size=9857) 2024-11-14T00:29:45,035 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/049bcc2d13b24b748df83b2192b2965b 2024-11-14T00:29:45,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/268395cfc62d4e5b9197f056bb5b16b7 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/268395cfc62d4e5b9197f056bb5b16b7 2024-11-14T00:29:45,042 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/268395cfc62d4e5b9197f056bb5b16b7, entries=100, sequenceid=403, filesize=9.6 K 2024-11-14T00:29:45,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/533acdddb20d4b3da650d1949312c8e6 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/533acdddb20d4b3da650d1949312c8e6 2024-11-14T00:29:45,048 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/533acdddb20d4b3da650d1949312c8e6, entries=100, sequenceid=403, filesize=9.6 K 2024-11-14T00:29:45,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/049bcc2d13b24b748df83b2192b2965b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/049bcc2d13b24b748df83b2192b2965b 2024-11-14T00:29:45,051 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/049bcc2d13b24b748df83b2192b2965b, entries=100, sequenceid=403, filesize=9.6 K 2024-11-14T00:29:45,052 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 93f27665e92d07bc413648d25770240d in 1401ms, sequenceid=403, compaction requested=false 2024-11-14T00:29:45,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:45,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:45,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-14T00:29:45,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-14T00:29:45,058 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-14T00:29:45,058 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4160 sec 2024-11-14T00:29:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:45,129 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:45,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 3.4230 sec 2024-11-14T00:29:45,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-14T00:29:45,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/c518634a0dde4e4d9b6644e0ed1b0ed1 is 50, key is test_row_0/A:col10/1731544185129/Put/seqid=0 2024-11-14T00:29:45,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742318_1494 (size=14741) 2024-11-14T00:29:45,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544245129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544245129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544245130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544245133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544245143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,147 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544245245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544245245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544245245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544245246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544245255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544245456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544245456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544245459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544245460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544245465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,521 INFO [Thread-1845 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1845. 2024-11-14T00:29:45,522 DEBUG [Thread-1845 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:45,522 DEBUG [Thread-1845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:45,522 DEBUG [Thread-1845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:45,522 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:45,522 INFO [Thread-1841 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1841. 2024-11-14T00:29:45,522 DEBUG [Thread-1841 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:45,522 DEBUG [Thread-1841 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:45,522 DEBUG [Thread-1841 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:45,523 INFO [Thread-1843 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1843. 2024-11-14T00:29:45,523 DEBUG [Thread-1843 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:45,523 DEBUG [Thread-1843 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:45,523 DEBUG [Thread-1843 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:45,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:45,523 INFO [Thread-1837 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1837. 2024-11-14T00:29:45,523 DEBUG [Thread-1837 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:45,523 DEBUG [Thread-1837 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:45,523 DEBUG [Thread-1837 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:45,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:45,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:45,531 INFO [Thread-1839 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1839. 2024-11-14T00:29:45,531 DEBUG [Thread-1839 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:45,531 DEBUG [Thread-1839 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:45,531 DEBUG [Thread-1839 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:45,531 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:45,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/c518634a0dde4e4d9b6644e0ed1b0ed1 2024-11-14T00:29:45,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/cc49e9b0f5a9414ea339c8a6a84058ce is 50, key is test_row_0/B:col10/1731544185129/Put/seqid=0 2024-11-14T00:29:45,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742319_1495 (size=12301) 2024-11-14T00:29:45,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544245766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544245767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544245775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544245775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544245775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T00:29:45,796 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:45,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/cc49e9b0f5a9414ea339c8a6a84058ce 2024-11-14T00:29:45,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/eb4d4fce4abd4435b8fe4edb82c1f477 is 50, key is test_row_0/C:col10/1731544185129/Put/seqid=0 2024-11-14T00:29:45,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742320_1496 (size=12301) 2024-11-14T00:29:46,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:46,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54670 deadline: 1731544246275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54726 deadline: 1731544246275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:46,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:46,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:46,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54720 deadline: 1731544246286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54684 deadline: 1731544246286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:54680 deadline: 1731544246285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 , the old value is region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., hostname=63baa11b75f3,33199,1731544045387, seqNum=2 because the exception is null or not the one we care about 2024-11-14T00:29:46,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/eb4d4fce4abd4435b8fe4edb82c1f477 2024-11-14T00:29:46,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/c518634a0dde4e4d9b6644e0ed1b0ed1 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/c518634a0dde4e4d9b6644e0ed1b0ed1 2024-11-14T00:29:46,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/c518634a0dde4e4d9b6644e0ed1b0ed1, entries=200, sequenceid=437, filesize=14.4 K 2024-11-14T00:29:46,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/cc49e9b0f5a9414ea339c8a6a84058ce as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/cc49e9b0f5a9414ea339c8a6a84058ce 2024-11-14T00:29:46,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/cc49e9b0f5a9414ea339c8a6a84058ce, entries=150, sequenceid=437, filesize=12.0 K 2024-11-14T00:29:46,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/eb4d4fce4abd4435b8fe4edb82c1f477 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/eb4d4fce4abd4435b8fe4edb82c1f477 2024-11-14T00:29:46,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/eb4d4fce4abd4435b8fe4edb82c1f477, entries=150, sequenceid=437, filesize=12.0 K 2024-11-14T00:29:46,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 93f27665e92d07bc413648d25770240d in 1253ms, sequenceid=437, compaction requested=true 2024-11-14T00:29:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:46,382 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:46,382 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 93f27665e92d07bc413648d25770240d:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:46,383 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37751 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:46,383 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:46,383 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/B is initiating minor compaction (all files) 2024-11-14T00:29:46,383 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/A is initiating minor compaction (all files) 2024-11-14T00:29:46,383 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/A in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:46,383 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/B in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:46,383 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1943a902ad3f4088b350834c2104a337, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/533acdddb20d4b3da650d1949312c8e6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/cc49e9b0f5a9414ea339c8a6a84058ce] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=34.5 K 2024-11-14T00:29:46,383 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/d77c6830c48b40ef914e9ece92133a1b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/268395cfc62d4e5b9197f056bb5b16b7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/c518634a0dde4e4d9b6644e0ed1b0ed1] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=36.9 K 2024-11-14T00:29:46,383 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting d77c6830c48b40ef914e9ece92133a1b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731544181956 2024-11-14T00:29:46,383 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 1943a902ad3f4088b350834c2104a337, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731544181956 2024-11-14T00:29:46,384 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 268395cfc62d4e5b9197f056bb5b16b7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731544183155 2024-11-14T00:29:46,384 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 533acdddb20d4b3da650d1949312c8e6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731544183155 2024-11-14T00:29:46,384 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting cc49e9b0f5a9414ea339c8a6a84058ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731544183840 2024-11-14T00:29:46,384 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting c518634a0dde4e4d9b6644e0ed1b0ed1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731544183840 2024-11-14T00:29:46,390 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#A#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:46,391 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/f825bd2ada4d45658f19d69274753171 is 50, key is test_row_0/A:col10/1731544185129/Put/seqid=0 2024-11-14T00:29:46,395 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#B#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:46,395 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/71c898ede8e847068685e1c6cb89e377 is 50, key is test_row_0/B:col10/1731544185129/Put/seqid=0 2024-11-14T00:29:46,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742321_1497 (size=13255) 2024-11-14T00:29:46,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742322_1498 (size=13255) 2024-11-14T00:29:46,809 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/f825bd2ada4d45658f19d69274753171 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/f825bd2ada4d45658f19d69274753171 2024-11-14T00:29:46,815 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/71c898ede8e847068685e1c6cb89e377 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/71c898ede8e847068685e1c6cb89e377 2024-11-14T00:29:46,831 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/B of 93f27665e92d07bc413648d25770240d into 71c898ede8e847068685e1c6cb89e377(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:46,831 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:46,831 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/B, priority=13, startTime=1731544186382; duration=0sec 2024-11-14T00:29:46,831 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:46,831 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:B 2024-11-14T00:29:46,831 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:46,834 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/A of 93f27665e92d07bc413648d25770240d into f825bd2ada4d45658f19d69274753171(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:46,834 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:46,834 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/A, priority=13, startTime=1731544186382; duration=0sec 2024-11-14T00:29:46,834 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:46,834 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:A 2024-11-14T00:29:46,844 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:46,844 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 93f27665e92d07bc413648d25770240d/C is initiating minor compaction (all files) 2024-11-14T00:29:46,844 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 93f27665e92d07bc413648d25770240d/C in TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:46,844 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7d548ec02b594b6c8d9f06f3c41289c8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/049bcc2d13b24b748df83b2192b2965b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/eb4d4fce4abd4435b8fe4edb82c1f477] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp, totalSize=34.5 K 2024-11-14T00:29:46,845 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d548ec02b594b6c8d9f06f3c41289c8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731544181956 2024-11-14T00:29:46,846 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 049bcc2d13b24b748df83b2192b2965b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731544183155 2024-11-14T00:29:46,846 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting eb4d4fce4abd4435b8fe4edb82c1f477, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731544183840 2024-11-14T00:29:46,857 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 93f27665e92d07bc413648d25770240d#C#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:46,858 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/9fff080e28ee4a9e98c5b277172abdfe is 50, key is test_row_0/C:col10/1731544185129/Put/seqid=0 2024-11-14T00:29:46,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742323_1499 (size=13255) 2024-11-14T00:29:47,283 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/9fff080e28ee4a9e98c5b277172abdfe as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/9fff080e28ee4a9e98c5b277172abdfe 2024-11-14T00:29:47,287 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 93f27665e92d07bc413648d25770240d/C of 93f27665e92d07bc413648d25770240d into 9fff080e28ee4a9e98c5b277172abdfe(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:47,287 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 93f27665e92d07bc413648d25770240d: 2024-11-14T00:29:47,287 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., storeName=93f27665e92d07bc413648d25770240d/C, priority=13, startTime=1731544186382; duration=0sec 2024-11-14T00:29:47,287 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:47,287 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 93f27665e92d07bc413648d25770240d:C 2024-11-14T00:29:47,298 INFO [Thread-1826 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1826. 2024-11-14T00:29:47,298 DEBUG [Thread-1826 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:47,298 DEBUG [Thread-1826 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:47,298 DEBUG [Thread-1826 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:47,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:47,298 INFO [Thread-1832 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1832. 2024-11-14T00:29:47,298 DEBUG [Thread-1832 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:47,298 DEBUG [Thread-1832 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:47,298 DEBUG [Thread-1832 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:47,298 INFO [Thread-1828 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1828. 2024-11-14T00:29:47,299 DEBUG [Thread-1828 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:47,299 DEBUG [Thread-1828 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:47,299 DEBUG [Thread-1828 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:47,299 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:47,299 INFO [Thread-1834 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1834. 2024-11-14T00:29:47,299 DEBUG [Thread-1834 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:47,299 DEBUG [Thread-1834 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:47,299 DEBUG [Thread-1834 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:47,299 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:47,300 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:47,306 INFO [Thread-1830 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1830. 2024-11-14T00:29:47,306 DEBUG [Thread-1830 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:29:47,306 DEBUG [Thread-1830 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:47,306 DEBUG [Thread-1830 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:47,306 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1013 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3038 rows 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1010 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3020 rows 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1019 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3056 rows 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1008 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3020 rows 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1006 2024-11-14T00:29:47,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3016 rows 2024-11-14T00:29:47,307 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T00:29:47,307 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T00:29:47,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:47,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:47,307 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:29:47,308 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-14T00:29:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T00:29:47,311 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544187310"}]},"ts":"1731544187310"} 2024-11-14T00:29:47,312 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T00:29:47,312 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T00:29:47,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T00:29:47,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=93f27665e92d07bc413648d25770240d, UNASSIGN}] 2024-11-14T00:29:47,314 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=93f27665e92d07bc413648d25770240d, UNASSIGN 2024-11-14T00:29:47,314 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=93f27665e92d07bc413648d25770240d, regionState=CLOSING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:47,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=93f27665e92d07bc413648d25770240d, UNASSIGN because future has completed 2024-11-14T00:29:47,316 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T00:29:47,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:29:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T00:29:47,468 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:47,468 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T00:29:47,468 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing 93f27665e92d07bc413648d25770240d, disabling compactions & flushes 2024-11-14T00:29:47,468 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:47,468 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:47,468 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. after waiting 0 ms 2024-11-14T00:29:47,468 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:47,468 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing 93f27665e92d07bc413648d25770240d 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-14T00:29:47,472 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/97d904240c4145128c647d5b1d4599fd is 50, key is test_row_0/A:col10/1731544187305/Put/seqid=0 2024-11-14T00:29:47,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742324_1500 (size=12301) 2024-11-14T00:29:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T00:29:47,876 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/97d904240c4145128c647d5b1d4599fd 2024-11-14T00:29:47,881 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/d83e37a94733470999da8e4a576c0375 is 50, key is test_row_0/B:col10/1731544187305/Put/seqid=0 2024-11-14T00:29:47,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742325_1501 (size=12301) 2024-11-14T00:29:47,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T00:29:48,285 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/d83e37a94733470999da8e4a576c0375 2024-11-14T00:29:48,291 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/58ccfa91de30403ea001195d6e87d9f4 is 50, key is test_row_0/C:col10/1731544187305/Put/seqid=0 2024-11-14T00:29:48,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742326_1502 (size=12301) 2024-11-14T00:29:48,296 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/58ccfa91de30403ea001195d6e87d9f4 2024-11-14T00:29:48,299 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/A/97d904240c4145128c647d5b1d4599fd as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/97d904240c4145128c647d5b1d4599fd 2024-11-14T00:29:48,301 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/97d904240c4145128c647d5b1d4599fd, entries=150, sequenceid=450, filesize=12.0 K 2024-11-14T00:29:48,302 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/B/d83e37a94733470999da8e4a576c0375 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/d83e37a94733470999da8e4a576c0375 2024-11-14T00:29:48,305 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/d83e37a94733470999da8e4a576c0375, entries=150, sequenceid=450, filesize=12.0 K 2024-11-14T00:29:48,305 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/.tmp/C/58ccfa91de30403ea001195d6e87d9f4 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/58ccfa91de30403ea001195d6e87d9f4 2024-11-14T00:29:48,308 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/58ccfa91de30403ea001195d6e87d9f4, entries=150, sequenceid=450, filesize=12.0 K 2024-11-14T00:29:48,309 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 93f27665e92d07bc413648d25770240d in 841ms, sequenceid=450, compaction requested=false 2024-11-14T00:29:48,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/f5c0b1799a8642948bff3cb80e498611, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/b4fc5f8b09cb4ed7b5972dad29d7dfe8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/2fa7d66fbb0743e28af5eef6aeb18bf0, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/126a012beca041a6aef54893ef9a03bb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/cea7e8deddfc494696adc7f1c3bb6603, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/a64aa20f2faa41ddb29bcbe8cd6ed7c4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/efe68c4794ba46a6b16e2329e5b5db95, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/d913e351257742b29f5504784eb667fb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/23f765f6b4484f13b80bf48253a0e286, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/06126d488f234c6e8f7d8f7e68bc330b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/a912bd8b86604819992dfa0ea7b29c0a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/823b6175595547f29c7e744142eb8b10, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/eb3afb3d4c14424dbb931ff84e4b719b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/4139f465f8144710b582647aa1829938, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/db3446552239428ebd9c8e2efc86b325, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/8351a188c9fe40b29c42f28704cb3b15, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/ae34fbecef72466a93a89a6a42273db5, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/7a418677706d454b8d81bb3f4a9ff84e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/24608c08eaa041dbad3accdcee8ce122, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/1216b70b992940c988c045f8b0e22848, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/bc761337c2fd4f7581b37455806c1328, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/2a5d002ae6784dada9dbb8d4c6391bb6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/aeebae8d7a664568b251694bee2782d7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/d6f96fbf78c44bc08213887a7e343f89, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/fa803008ae3a4737a8f712807681bcaf, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/d77c6830c48b40ef914e9ece92133a1b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/268395cfc62d4e5b9197f056bb5b16b7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/c518634a0dde4e4d9b6644e0ed1b0ed1] to archive 2024-11-14T00:29:48,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:29:48,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/f5c0b1799a8642948bff3cb80e498611 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/f5c0b1799a8642948bff3cb80e498611 2024-11-14T00:29:48,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/b4fc5f8b09cb4ed7b5972dad29d7dfe8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/b4fc5f8b09cb4ed7b5972dad29d7dfe8 2024-11-14T00:29:48,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/2fa7d66fbb0743e28af5eef6aeb18bf0 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/2fa7d66fbb0743e28af5eef6aeb18bf0 2024-11-14T00:29:48,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/126a012beca041a6aef54893ef9a03bb to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/126a012beca041a6aef54893ef9a03bb 2024-11-14T00:29:48,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/cea7e8deddfc494696adc7f1c3bb6603 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/cea7e8deddfc494696adc7f1c3bb6603 2024-11-14T00:29:48,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/a64aa20f2faa41ddb29bcbe8cd6ed7c4 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/a64aa20f2faa41ddb29bcbe8cd6ed7c4 2024-11-14T00:29:48,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/efe68c4794ba46a6b16e2329e5b5db95 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/efe68c4794ba46a6b16e2329e5b5db95 2024-11-14T00:29:48,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/d913e351257742b29f5504784eb667fb to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/d913e351257742b29f5504784eb667fb 2024-11-14T00:29:48,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/23f765f6b4484f13b80bf48253a0e286 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/23f765f6b4484f13b80bf48253a0e286 2024-11-14T00:29:48,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/06126d488f234c6e8f7d8f7e68bc330b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/06126d488f234c6e8f7d8f7e68bc330b 2024-11-14T00:29:48,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/a912bd8b86604819992dfa0ea7b29c0a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/a912bd8b86604819992dfa0ea7b29c0a 2024-11-14T00:29:48,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/823b6175595547f29c7e744142eb8b10 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/823b6175595547f29c7e744142eb8b10 2024-11-14T00:29:48,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/eb3afb3d4c14424dbb931ff84e4b719b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/eb3afb3d4c14424dbb931ff84e4b719b 2024-11-14T00:29:48,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/4139f465f8144710b582647aa1829938 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/4139f465f8144710b582647aa1829938 2024-11-14T00:29:48,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/db3446552239428ebd9c8e2efc86b325 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/db3446552239428ebd9c8e2efc86b325 2024-11-14T00:29:48,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/8351a188c9fe40b29c42f28704cb3b15 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/8351a188c9fe40b29c42f28704cb3b15 2024-11-14T00:29:48,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/ae34fbecef72466a93a89a6a42273db5 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/ae34fbecef72466a93a89a6a42273db5 2024-11-14T00:29:48,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/7a418677706d454b8d81bb3f4a9ff84e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/7a418677706d454b8d81bb3f4a9ff84e 2024-11-14T00:29:48,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/24608c08eaa041dbad3accdcee8ce122 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/24608c08eaa041dbad3accdcee8ce122 2024-11-14T00:29:48,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/1216b70b992940c988c045f8b0e22848 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/1216b70b992940c988c045f8b0e22848 2024-11-14T00:29:48,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/bc761337c2fd4f7581b37455806c1328 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/bc761337c2fd4f7581b37455806c1328 2024-11-14T00:29:48,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/2a5d002ae6784dada9dbb8d4c6391bb6 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/2a5d002ae6784dada9dbb8d4c6391bb6 2024-11-14T00:29:48,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/aeebae8d7a664568b251694bee2782d7 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/aeebae8d7a664568b251694bee2782d7 2024-11-14T00:29:48,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/d6f96fbf78c44bc08213887a7e343f89 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/d6f96fbf78c44bc08213887a7e343f89 2024-11-14T00:29:48,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/fa803008ae3a4737a8f712807681bcaf to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/fa803008ae3a4737a8f712807681bcaf 2024-11-14T00:29:48,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/d77c6830c48b40ef914e9ece92133a1b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/d77c6830c48b40ef914e9ece92133a1b 2024-11-14T00:29:48,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/268395cfc62d4e5b9197f056bb5b16b7 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/268395cfc62d4e5b9197f056bb5b16b7 2024-11-14T00:29:48,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/c518634a0dde4e4d9b6644e0ed1b0ed1 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/c518634a0dde4e4d9b6644e0ed1b0ed1 2024-11-14T00:29:48,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/186c49d849b94e6c864295f764248c5b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/16a67c68d3ac4265a51a708399f13e44, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/a81120bff41c479dabaacb7f402fe965, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/2af7fffbdf1e4635be6e6b3d16132b0c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/3268136d5fa64a83a261e52850dd68bb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/4da4b23ffd19426dbdcfd5037c6bd073, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/0cbeaa10039f484399acf6d1de333745, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/e27de5ab1d464fd4983329ce9734f036, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/758d7dd144744037a60112bdc36db214, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/da9df339bf884c95bd5e1f04bc795582, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1695b88d7e8048b2a0a74f2a4c47903d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/ea4ecfdd382944caa2d2edec7bbaff3a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/9f27cc09586c4c73a106ea49b966c8fe, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/8f5b225aadfd4959b29e9990ccb3c7d2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/a25935f0962e4deaa34fb186618de34e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/879d9ec3074440ab979d00c2eec6193f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/c89894b6e49b424980e64686d0b4575f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/3ad8a42f71ff438ebf86139d247779ef, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/73b028b37fb44771a30fa0c440ca9cae, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1f10b0643b0f41319972f6c38459e522, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/7a0798d4c3f44b818af061f319b1e612, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/b009d4154f854b3c80a13eb6e0185610, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/7c54685fced1440a905b06f7b11a8b11, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1fb199bbe67c42f6ba0330d1a437e494, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1943a902ad3f4088b350834c2104a337, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/6107ba4f6f8f425bb52b28db98d52071, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/533acdddb20d4b3da650d1949312c8e6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/cc49e9b0f5a9414ea339c8a6a84058ce] to archive 2024-11-14T00:29:48,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:29:48,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/186c49d849b94e6c864295f764248c5b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/186c49d849b94e6c864295f764248c5b 2024-11-14T00:29:48,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/16a67c68d3ac4265a51a708399f13e44 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/16a67c68d3ac4265a51a708399f13e44 2024-11-14T00:29:48,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/a81120bff41c479dabaacb7f402fe965 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/a81120bff41c479dabaacb7f402fe965 2024-11-14T00:29:48,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/2af7fffbdf1e4635be6e6b3d16132b0c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/2af7fffbdf1e4635be6e6b3d16132b0c 2024-11-14T00:29:48,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/3268136d5fa64a83a261e52850dd68bb to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/3268136d5fa64a83a261e52850dd68bb 2024-11-14T00:29:48,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/4da4b23ffd19426dbdcfd5037c6bd073 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/4da4b23ffd19426dbdcfd5037c6bd073 2024-11-14T00:29:48,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/0cbeaa10039f484399acf6d1de333745 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/0cbeaa10039f484399acf6d1de333745 2024-11-14T00:29:48,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/e27de5ab1d464fd4983329ce9734f036 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/e27de5ab1d464fd4983329ce9734f036 2024-11-14T00:29:48,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/758d7dd144744037a60112bdc36db214 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/758d7dd144744037a60112bdc36db214 2024-11-14T00:29:48,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/da9df339bf884c95bd5e1f04bc795582 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/da9df339bf884c95bd5e1f04bc795582 2024-11-14T00:29:48,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1695b88d7e8048b2a0a74f2a4c47903d to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1695b88d7e8048b2a0a74f2a4c47903d 2024-11-14T00:29:48,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/ea4ecfdd382944caa2d2edec7bbaff3a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/ea4ecfdd382944caa2d2edec7bbaff3a 2024-11-14T00:29:48,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/9f27cc09586c4c73a106ea49b966c8fe to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/9f27cc09586c4c73a106ea49b966c8fe 2024-11-14T00:29:48,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/8f5b225aadfd4959b29e9990ccb3c7d2 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/8f5b225aadfd4959b29e9990ccb3c7d2 2024-11-14T00:29:48,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/a25935f0962e4deaa34fb186618de34e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/a25935f0962e4deaa34fb186618de34e 2024-11-14T00:29:48,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/879d9ec3074440ab979d00c2eec6193f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/879d9ec3074440ab979d00c2eec6193f 2024-11-14T00:29:48,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/c89894b6e49b424980e64686d0b4575f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/c89894b6e49b424980e64686d0b4575f 2024-11-14T00:29:48,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/3ad8a42f71ff438ebf86139d247779ef to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/3ad8a42f71ff438ebf86139d247779ef 2024-11-14T00:29:48,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/73b028b37fb44771a30fa0c440ca9cae to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/73b028b37fb44771a30fa0c440ca9cae 2024-11-14T00:29:48,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1f10b0643b0f41319972f6c38459e522 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1f10b0643b0f41319972f6c38459e522 2024-11-14T00:29:48,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/7a0798d4c3f44b818af061f319b1e612 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/7a0798d4c3f44b818af061f319b1e612 2024-11-14T00:29:48,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/b009d4154f854b3c80a13eb6e0185610 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/b009d4154f854b3c80a13eb6e0185610 2024-11-14T00:29:48,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/7c54685fced1440a905b06f7b11a8b11 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/7c54685fced1440a905b06f7b11a8b11 2024-11-14T00:29:48,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1fb199bbe67c42f6ba0330d1a437e494 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1fb199bbe67c42f6ba0330d1a437e494 2024-11-14T00:29:48,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1943a902ad3f4088b350834c2104a337 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/1943a902ad3f4088b350834c2104a337 2024-11-14T00:29:48,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/6107ba4f6f8f425bb52b28db98d52071 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/6107ba4f6f8f425bb52b28db98d52071 2024-11-14T00:29:48,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/533acdddb20d4b3da650d1949312c8e6 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/533acdddb20d4b3da650d1949312c8e6 2024-11-14T00:29:48,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/cc49e9b0f5a9414ea339c8a6a84058ce to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/cc49e9b0f5a9414ea339c8a6a84058ce 2024-11-14T00:29:48,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/27e1925ef9664e5ab6958d5e82d1d346, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/272f1fdb574f47fc80f78c7deecb88ca, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/d7d01cbdbdd74bddbe826236cdd6a056, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/8dd97f24f85041d1a0f9ea76edbae536, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4e61b526e91147c9863df684455bb356, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/d47d677bcbd54cc5a19c504d6ccf8917, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4d08f7a1c2b64b0db60d548ea5973fb3, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/9cf228b96462420e87161dc39c6378ed, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/be0cb009154f463ca29e2a433d5486a6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/5d783a3b284e455ba5dfa6a634c29f8b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/8ab3705c0cb94a04ab69e4b830acc305, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/41c2989136a84e17bc16e86bae6de5c7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/c426188b545f421d9bb077563713480b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/3dffbde0d9f64201b8a8eb86d163ef46, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/2b45f693ed2e49c2b44c058f8c03b1ab, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/25cb9a88dd6b4646813856e3b143133e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/383f5f66b581466ab7287dc6b0efe03b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7f75dde3e052404fbcd2bfd60ce4b713, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7f28540e95ac4a08ae2d4824ae098647, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/b97ac737643442d3a45694dc10a1b178, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/2132940e1a7a45e9b146c677001f3628, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/eadc00944d5943e592fe154b2744fc8a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7e1c111dc2264afd957b08a0fa85cc71, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/46bfa3a17e8a476284435ff6af074c2d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7d548ec02b594b6c8d9f06f3c41289c8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/9c357a70ab404f1e95d0bf27589f7f10, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/049bcc2d13b24b748df83b2192b2965b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/eb4d4fce4abd4435b8fe4edb82c1f477] to archive 2024-11-14T00:29:48,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:29:48,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/27e1925ef9664e5ab6958d5e82d1d346 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/27e1925ef9664e5ab6958d5e82d1d346 2024-11-14T00:29:48,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/272f1fdb574f47fc80f78c7deecb88ca to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/272f1fdb574f47fc80f78c7deecb88ca 2024-11-14T00:29:48,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/d7d01cbdbdd74bddbe826236cdd6a056 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/d7d01cbdbdd74bddbe826236cdd6a056 2024-11-14T00:29:48,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/8dd97f24f85041d1a0f9ea76edbae536 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/8dd97f24f85041d1a0f9ea76edbae536 2024-11-14T00:29:48,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4e61b526e91147c9863df684455bb356 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4e61b526e91147c9863df684455bb356 2024-11-14T00:29:48,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/d47d677bcbd54cc5a19c504d6ccf8917 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/d47d677bcbd54cc5a19c504d6ccf8917 2024-11-14T00:29:48,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4d08f7a1c2b64b0db60d548ea5973fb3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/4d08f7a1c2b64b0db60d548ea5973fb3 2024-11-14T00:29:48,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/9cf228b96462420e87161dc39c6378ed to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/9cf228b96462420e87161dc39c6378ed 2024-11-14T00:29:48,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/be0cb009154f463ca29e2a433d5486a6 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/be0cb009154f463ca29e2a433d5486a6 2024-11-14T00:29:48,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/5d783a3b284e455ba5dfa6a634c29f8b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/5d783a3b284e455ba5dfa6a634c29f8b 2024-11-14T00:29:48,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/8ab3705c0cb94a04ab69e4b830acc305 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/8ab3705c0cb94a04ab69e4b830acc305 2024-11-14T00:29:48,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/41c2989136a84e17bc16e86bae6de5c7 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/41c2989136a84e17bc16e86bae6de5c7 2024-11-14T00:29:48,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/c426188b545f421d9bb077563713480b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/c426188b545f421d9bb077563713480b 2024-11-14T00:29:48,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/3dffbde0d9f64201b8a8eb86d163ef46 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/3dffbde0d9f64201b8a8eb86d163ef46 2024-11-14T00:29:48,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/2b45f693ed2e49c2b44c058f8c03b1ab to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/2b45f693ed2e49c2b44c058f8c03b1ab 2024-11-14T00:29:48,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/25cb9a88dd6b4646813856e3b143133e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/25cb9a88dd6b4646813856e3b143133e 2024-11-14T00:29:48,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/383f5f66b581466ab7287dc6b0efe03b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/383f5f66b581466ab7287dc6b0efe03b 2024-11-14T00:29:48,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7f75dde3e052404fbcd2bfd60ce4b713 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7f75dde3e052404fbcd2bfd60ce4b713 2024-11-14T00:29:48,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7f28540e95ac4a08ae2d4824ae098647 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7f28540e95ac4a08ae2d4824ae098647 2024-11-14T00:29:48,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/b97ac737643442d3a45694dc10a1b178 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/b97ac737643442d3a45694dc10a1b178 2024-11-14T00:29:48,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/2132940e1a7a45e9b146c677001f3628 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/2132940e1a7a45e9b146c677001f3628 2024-11-14T00:29:48,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/eadc00944d5943e592fe154b2744fc8a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/eadc00944d5943e592fe154b2744fc8a 2024-11-14T00:29:48,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7e1c111dc2264afd957b08a0fa85cc71 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7e1c111dc2264afd957b08a0fa85cc71 2024-11-14T00:29:48,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/46bfa3a17e8a476284435ff6af074c2d to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/46bfa3a17e8a476284435ff6af074c2d 2024-11-14T00:29:48,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7d548ec02b594b6c8d9f06f3c41289c8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/7d548ec02b594b6c8d9f06f3c41289c8 2024-11-14T00:29:48,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/9c357a70ab404f1e95d0bf27589f7f10 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/9c357a70ab404f1e95d0bf27589f7f10 2024-11-14T00:29:48,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/049bcc2d13b24b748df83b2192b2965b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/049bcc2d13b24b748df83b2192b2965b 2024-11-14T00:29:48,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/eb4d4fce4abd4435b8fe4edb82c1f477 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/eb4d4fce4abd4435b8fe4edb82c1f477 2024-11-14T00:29:48,391 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/recovered.edits/453.seqid, newMaxSeqId=453, maxSeqId=1 2024-11-14T00:29:48,392 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. 2024-11-14T00:29:48,392 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for 93f27665e92d07bc413648d25770240d: Waiting for close lock at 1731544187468Running coprocessor pre-close hooks at 1731544187468Disabling compacts and flushes for region at 1731544187468Disabling writes for close at 1731544187468Obtaining lock to block concurrent updates at 1731544187468Preparing flush snapshotting stores in 93f27665e92d07bc413648d25770240d at 1731544187468Finished memstore snapshotting TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d., syncing WAL and waiting on mvcc, flushsize=dataSize=48090, getHeapSize=126720, getOffHeapSize=0, getCellsCount=1050 at 1731544187468Flushing stores of TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d. at 1731544187469 (+1 ms)Flushing 93f27665e92d07bc413648d25770240d/A: creating writer at 1731544187469Flushing 93f27665e92d07bc413648d25770240d/A: appending metadata at 1731544187472 (+3 ms)Flushing 93f27665e92d07bc413648d25770240d/A: closing flushed file at 1731544187472Flushing 93f27665e92d07bc413648d25770240d/B: creating writer at 1731544187879 (+407 ms)Flushing 93f27665e92d07bc413648d25770240d/B: appending metadata at 1731544187881 (+2 ms)Flushing 93f27665e92d07bc413648d25770240d/B: closing flushed file at 1731544187881Flushing 93f27665e92d07bc413648d25770240d/C: creating writer at 1731544188288 (+407 ms)Flushing 93f27665e92d07bc413648d25770240d/C: appending metadata at 1731544188291 (+3 ms)Flushing 93f27665e92d07bc413648d25770240d/C: closing flushed file at 1731544188291Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a96afd0: reopening flushed file at 1731544188298 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23c7f73a: reopening flushed file at 1731544188302 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@723a31b6: reopening flushed file at 1731544188305 (+3 ms)Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 93f27665e92d07bc413648d25770240d in 841ms, sequenceid=450, compaction requested=false at 1731544188309 (+4 ms)Writing region close event to WAL at 1731544188388 (+79 ms)Running coprocessor post-close hooks at 1731544188392 (+4 ms)Closed at 1731544188392 2024-11-14T00:29:48,394 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed 93f27665e92d07bc413648d25770240d 2024-11-14T00:29:48,395 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=93f27665e92d07bc413648d25770240d, regionState=CLOSED 2024-11-14T00:29:48,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:29:48,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-14T00:29:48,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure 93f27665e92d07bc413648d25770240d, server=63baa11b75f3,33199,1731544045387 in 1.0800 sec 2024-11-14T00:29:48,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-14T00:29:48,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=93f27665e92d07bc413648d25770240d, UNASSIGN in 1.0850 sec 2024-11-14T00:29:48,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-14T00:29:48,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.0870 sec 2024-11-14T00:29:48,401 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544188401"}]},"ts":"1731544188401"} 2024-11-14T00:29:48,403 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T00:29:48,403 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T00:29:48,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.0950 sec 2024-11-14T00:29:48,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T00:29:48,445 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:48,445 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-14T00:29:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:48,447 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:48,447 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T00:29:48,450 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d 2024-11-14T00:29:48,452 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/recovered.edits] 2024-11-14T00:29:48,454 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/97d904240c4145128c647d5b1d4599fd to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/97d904240c4145128c647d5b1d4599fd 2024-11-14T00:29:48,455 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/f825bd2ada4d45658f19d69274753171 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/A/f825bd2ada4d45658f19d69274753171 2024-11-14T00:29:48,456 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/71c898ede8e847068685e1c6cb89e377 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/71c898ede8e847068685e1c6cb89e377 2024-11-14T00:29:48,457 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/d83e37a94733470999da8e4a576c0375 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/B/d83e37a94733470999da8e4a576c0375 2024-11-14T00:29:48,458 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/58ccfa91de30403ea001195d6e87d9f4 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/58ccfa91de30403ea001195d6e87d9f4 2024-11-14T00:29:48,459 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/9fff080e28ee4a9e98c5b277172abdfe to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/C/9fff080e28ee4a9e98c5b277172abdfe 2024-11-14T00:29:48,461 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/recovered.edits/453.seqid to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d/recovered.edits/453.seqid 2024-11-14T00:29:48,461 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/93f27665e92d07bc413648d25770240d 2024-11-14T00:29:48,461 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T00:29:48,463 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:48,464 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T00:29:48,466 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T00:29:48,467 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:48,467 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T00:29:48,467 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731544188467"}]},"ts":"9223372036854775807"} 2024-11-14T00:29:48,468 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T00:29:48,469 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 93f27665e92d07bc413648d25770240d, NAME => 'TestAcidGuarantees,,1731544163232.93f27665e92d07bc413648d25770240d.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T00:29:48,469 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T00:29:48,469 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731544188469"}]},"ts":"9223372036854775807"} 2024-11-14T00:29:48,470 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T00:29:48,470 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:48,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 25 msec 2024-11-14T00:29:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T00:29:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T00:29:48,558 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:48,565 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=239 (was 240), OpenFileDescriptor=449 (was 463), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1150 (was 1272), ProcessCount=11 (was 11), AvailableMemoryMB=3315 (was 2748) - AvailableMemoryMB LEAK? - 2024-11-14T00:29:48,573 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=1150, ProcessCount=11, AvailableMemoryMB=3315 2024-11-14T00:29:48,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T00:29:48,574 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T00:29:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:48,575 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T00:29:48,575 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:48,575 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-11-14T00:29:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T00:29:48,576 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T00:29:48,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742327_1503 (size=959) 2024-11-14T00:29:48,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T00:29:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T00:29:48,984 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c 2024-11-14T00:29:48,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742328_1504 (size=53) 2024-11-14T00:29:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T00:29:49,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:29:49,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4779df6820910ec85365292104f48afc, disabling compactions & flushes 2024-11-14T00:29:49,393 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:49,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:49,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. after waiting 0 ms 2024-11-14T00:29:49,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:49,393 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:49,393 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4779df6820910ec85365292104f48afc: Waiting for close lock at 1731544189393Disabling compacts and flushes for region at 1731544189393Disabling writes for close at 1731544189393Writing region close event to WAL at 1731544189393Closed at 1731544189393 2024-11-14T00:29:49,394 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T00:29:49,394 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731544189394"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731544189394"}]},"ts":"1731544189394"} 2024-11-14T00:29:49,395 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T00:29:49,396 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T00:29:49,396 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544189396"}]},"ts":"1731544189396"} 2024-11-14T00:29:49,397 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T00:29:49,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4779df6820910ec85365292104f48afc, ASSIGN}] 2024-11-14T00:29:49,398 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4779df6820910ec85365292104f48afc, ASSIGN 2024-11-14T00:29:49,399 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4779df6820910ec85365292104f48afc, ASSIGN; state=OFFLINE, location=63baa11b75f3,33199,1731544045387; forceNewPlan=false, retain=false 2024-11-14T00:29:49,550 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=4779df6820910ec85365292104f48afc, regionState=OPENING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:49,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4779df6820910ec85365292104f48afc, ASSIGN because future has completed 2024-11-14T00:29:49,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:29:49,705 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:49,705 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} 2024-11-14T00:29:49,705 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4779df6820910ec85365292104f48afc 2024-11-14T00:29:49,705 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:29:49,705 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for 4779df6820910ec85365292104f48afc 2024-11-14T00:29:49,705 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for 4779df6820910ec85365292104f48afc 2024-11-14T00:29:49,707 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4779df6820910ec85365292104f48afc 2024-11-14T00:29:49,708 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4779df6820910ec85365292104f48afc columnFamilyName A 2024-11-14T00:29:49,708 DEBUG [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:49,708 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] regionserver.HStore(327): Store=4779df6820910ec85365292104f48afc/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:29:49,708 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4779df6820910ec85365292104f48afc 2024-11-14T00:29:49,709 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4779df6820910ec85365292104f48afc columnFamilyName B 2024-11-14T00:29:49,710 DEBUG [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:49,710 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] regionserver.HStore(327): Store=4779df6820910ec85365292104f48afc/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:29:49,710 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4779df6820910ec85365292104f48afc 2024-11-14T00:29:49,711 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4779df6820910ec85365292104f48afc columnFamilyName C 2024-11-14T00:29:49,711 DEBUG [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:49,711 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] regionserver.HStore(327): Store=4779df6820910ec85365292104f48afc/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:29:49,712 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for 4779df6820910ec85365292104f48afc 2024-11-14T00:29:49,712 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc 2024-11-14T00:29:49,712 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc 2024-11-14T00:29:49,713 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for 4779df6820910ec85365292104f48afc 2024-11-14T00:29:49,713 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for 4779df6820910ec85365292104f48afc 2024-11-14T00:29:49,713 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T00:29:49,714 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for 4779df6820910ec85365292104f48afc 2024-11-14T00:29:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T00:29:49,716 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T00:29:49,716 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened 4779df6820910ec85365292104f48afc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72533184, jitterRate=0.08082866668701172}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T00:29:49,716 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4779df6820910ec85365292104f48afc 2024-11-14T00:29:49,716 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for 4779df6820910ec85365292104f48afc: Running coprocessor pre-open hook at 1731544189705Writing region info on filesystem at 1731544189705Initializing all the Stores at 1731544189706 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544189706Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544189706Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544189706Cleaning up temporary data from old regions at 1731544189713 (+7 ms)Running coprocessor post-open hooks at 1731544189716 (+3 ms)Region opened successfully at 1731544189716 2024-11-14T00:29:49,717 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., pid=138, masterSystemTime=1731544189702 2024-11-14T00:29:49,718 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:49,719 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:49,719 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=4779df6820910ec85365292104f48afc, regionState=OPEN, openSeqNum=2, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:49,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:29:49,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-14T00:29:49,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 in 170 msec 2024-11-14T00:29:49,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-14T00:29:49,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4779df6820910ec85365292104f48afc, ASSIGN in 325 msec 2024-11-14T00:29:49,725 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T00:29:49,725 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544189725"}]},"ts":"1731544189725"} 2024-11-14T00:29:49,726 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T00:29:49,727 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T00:29:49,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1530 sec 2024-11-14T00:29:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-14T00:29:50,726 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:50,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6302c151, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:50,727 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:50,727 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:50,727 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5562117e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:50,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:50,729 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dc8873c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:50,730 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:50,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:50,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:50,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:50,732 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T00:29:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 63baa11b75f3,36271,1731544044347 2024-11-14T00:29:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@299e17d3 2024-11-14T00:29:50,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T00:29:50,733 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T00:29:50,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T00:29:50,735 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T00:29:50,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T00:29:50,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742329_1505 (size=995) 2024-11-14T00:29:51,146 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-14T00:29:51,146 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-14T00:29:51,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T00:29:51,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4779df6820910ec85365292104f48afc, REOPEN/MOVE}] 2024-11-14T00:29:51,152 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4779df6820910ec85365292104f48afc, REOPEN/MOVE 2024-11-14T00:29:51,153 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=4779df6820910ec85365292104f48afc, regionState=CLOSING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:51,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4779df6820910ec85365292104f48afc, REOPEN/MOVE because future has completed 2024-11-14T00:29:51,155 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T00:29:51,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:29:51,307 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,307 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T00:29:51,307 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 4779df6820910ec85365292104f48afc, disabling compactions & flushes 2024-11-14T00:29:51,307 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:51,308 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:51,308 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. after waiting 0 ms 2024-11-14T00:29:51,308 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:51,313 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T00:29:51,314 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:51,314 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 4779df6820910ec85365292104f48afc: Waiting for close lock at 1731544191307Running coprocessor pre-close hooks at 1731544191307Disabling compacts and flushes for region at 1731544191307Disabling writes for close at 1731544191308 (+1 ms)Writing region close event to WAL at 1731544191308Running coprocessor post-close hooks at 1731544191314 (+6 ms)Closed at 1731544191314 2024-11-14T00:29:51,315 WARN [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: 4779df6820910ec85365292104f48afc to self. 2024-11-14T00:29:51,317 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,318 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=4779df6820910ec85365292104f48afc, regionState=CLOSED 2024-11-14T00:29:51,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:29:51,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-14T00:29:51,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 in 165 msec 2024-11-14T00:29:51,323 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4779df6820910ec85365292104f48afc, REOPEN/MOVE; state=CLOSED, location=63baa11b75f3,33199,1731544045387; forceNewPlan=false, retain=true 2024-11-14T00:29:51,474 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=4779df6820910ec85365292104f48afc, regionState=OPENING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:51,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4779df6820910ec85365292104f48afc, REOPEN/MOVE because future has completed 2024-11-14T00:29:51,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:29:51,631 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:51,631 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} 2024-11-14T00:29:51,632 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,632 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T00:29:51,632 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for 4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,632 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for 4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,634 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,635 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4779df6820910ec85365292104f48afc columnFamilyName A 2024-11-14T00:29:51,637 DEBUG [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:51,638 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] regionserver.HStore(327): Store=4779df6820910ec85365292104f48afc/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:29:51,638 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,639 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4779df6820910ec85365292104f48afc columnFamilyName B 2024-11-14T00:29:51,639 DEBUG [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:51,640 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] regionserver.HStore(327): Store=4779df6820910ec85365292104f48afc/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:29:51,640 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,641 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4779df6820910ec85365292104f48afc columnFamilyName C 2024-11-14T00:29:51,641 DEBUG [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:51,642 INFO [StoreOpener-4779df6820910ec85365292104f48afc-1 {}] regionserver.HStore(327): Store=4779df6820910ec85365292104f48afc/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T00:29:51,642 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for 4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,643 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,644 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,646 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for 4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,646 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for 4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,646 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T00:29:51,647 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for 4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,648 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened 4779df6820910ec85365292104f48afc; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69881919, jitterRate=0.04132173955440521}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T00:29:51,648 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,648 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for 4779df6820910ec85365292104f48afc: Running coprocessor pre-open hook at 1731544191632Writing region info on filesystem at 1731544191632Initializing all the Stores at 1731544191633 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544191633Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544191634 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731544191634Cleaning up temporary data from old regions at 1731544191646 (+12 ms)Running coprocessor post-open hooks at 1731544191648 (+2 ms)Region opened successfully at 1731544191648 2024-11-14T00:29:51,649 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., pid=143, masterSystemTime=1731544191627 2024-11-14T00:29:51,650 DEBUG [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:51,650 INFO [RS_OPEN_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:51,651 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=4779df6820910ec85365292104f48afc, regionState=OPEN, openSeqNum=5, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:51,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:29:51,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-11-14T00:29:51,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 in 176 msec 2024-11-14T00:29:51,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-14T00:29:51,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4779df6820910ec85365292104f48afc, REOPEN/MOVE in 503 msec 2024-11-14T00:29:51,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-14T00:29:51,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 507 msec 2024-11-14T00:29:51,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 922 msec 2024-11-14T00:29:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-14T00:29:51,660 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:51,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fec01c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,660 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:51,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:51,661 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:51,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6adac034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:51,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:51,663 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:51,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5775beef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e177505, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:51,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:51,664 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d447a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:51,666 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:51,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f0b288, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c494dd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:51,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:51,668 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:51,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:51,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:51,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35f40970, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:51,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:51,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:51,669 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c9dc4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a35b97a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:51,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:51,671 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aadc59d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:51,673 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:51,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d651e5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a0899da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:51,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:51,675 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@293a7e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:51,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:51,676 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:51,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ba6f318, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@325cce9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:51,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:51,678 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:51,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:51,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:51,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a3099a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:51,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:51,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:51,680 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:51,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1452644f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763f45be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:51,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:51,682 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:51,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59f063a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:51,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:51,684 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:51,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5058d404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0c0fb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:51,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:51,687 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41698de6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:51,688 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:51,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a9dcc4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@461194fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,690 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:51,690 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:51,691 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14562d98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:51,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:51,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:51,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@331c2c71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73d090d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 63baa11b75f3,36271,-1 for getting cluster id 2024-11-14T00:29:51,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T00:29:51,695 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65c5933d-f8ea-4873-8a51-cc20325705b8' 2024-11-14T00:29:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T00:29:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65c5933d-f8ea-4873-8a51-cc20325705b8" 2024-11-14T00:29:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ded3d50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [63baa11b75f3,36271,-1] 2024-11-14T00:29:51,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T00:29:51,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:29:51,696 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T00:29:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@606bab2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T00:29:51,697 DEBUG [Thread-2262 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:51,698 DEBUG [Thread-2264 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:51,698 DEBUG [Thread-2270 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:51,698 DEBUG [Thread-2266 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:51,698 DEBUG [Thread-2268 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:51,698 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:29:51,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:51,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:51,699 DEBUG [Thread-2277 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:51,699 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:51,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:51,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:51,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:51,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-14T00:29:51,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:51,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:51,701 DEBUG [Thread-2273 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:29:51,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:51,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:51,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:51,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:51,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:29:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T00:29:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:29:51,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:51,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:29:51,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:51,705 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:29:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:29:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:29:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:29:51,707 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:29:51,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:29:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,710 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:51,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T00:29:51,716 DEBUG [Thread-2281 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:51,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:51,719 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:51,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:29:51,722 DEBUG [Thread-2279 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:51,723 DEBUG [Thread-2279 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:51,723 DEBUG [Thread-2279 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:51,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:51,727 DEBUG [Thread-2279 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:29:51,730 DEBUG [Thread-2275 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T00:29:51,731 DEBUG [Thread-2275 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=63baa11b75f3,33199,1731544045387, seqNum=-1] 2024-11-14T00:29:51,731 DEBUG [Thread-2275 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T00:29:51,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T00:29:51,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544251732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:51,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544251732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5] 2024-11-14T00:29:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:51,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544251734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:51,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544251737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544251738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:51,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144d0392c4dd924b33880892cc9d56bdbc_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544191709/Put/seqid=0 2024-11-14T00:29:51,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742330_1506 (size=12154) 2024-11-14T00:29:51,784 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:51,805 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144d0392c4dd924b33880892cc9d56bdbc_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144d0392c4dd924b33880892cc9d56bdbc_4779df6820910ec85365292104f48afc 2024-11-14T00:29:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T00:29:51,809 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/8c258ad32d3a4778a2cfbf0b587dcd9e, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:29:51,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/8c258ad32d3a4778a2cfbf0b587dcd9e is 175, key is test_row_0/A:col10/1731544191709/Put/seqid=0 2024-11-14T00:29:51,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742331_1507 (size=30955) 2024-11-14T00:29:51,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544251846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:51,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:51,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544251846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:51,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544251846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:51,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544251846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544251846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:51,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:51,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T00:29:51,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:51,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:51,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:51,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:51,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T00:29:52,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T00:29:52,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:52,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:52,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:52,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544252055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:52,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544252056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:52,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544252056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:52,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:52,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544252060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:52,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:52,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544252060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:52,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:52,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:52,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T00:29:52,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:52,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:52,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:52,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,231 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=18, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/8c258ad32d3a4778a2cfbf0b587dcd9e 2024-11-14T00:29:52,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/fc9ab799c11145a995265e75b2737323 is 50, key is test_row_0/B:col10/1731544191709/Put/seqid=0 2024-11-14T00:29:52,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742332_1508 (size=12001) 2024-11-14T00:29:52,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T00:29:52,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:52,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:52,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T00:29:52,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544252367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:52,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544252367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:52,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544252367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:52,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544252367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:52,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544252376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:52,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T00:29:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:52,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T00:29:52,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:52,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:52,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:52,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/fc9ab799c11145a995265e75b2737323 2024-11-14T00:29:52,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/5bab2d396e5b4597bc2fb3d5fe415994 is 50, key is test_row_0/C:col10/1731544191709/Put/seqid=0 2024-11-14T00:29:52,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742333_1509 (size=12001) 2024-11-14T00:29:52,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/5bab2d396e5b4597bc2fb3d5fe415994 2024-11-14T00:29:52,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/8c258ad32d3a4778a2cfbf0b587dcd9e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8c258ad32d3a4778a2cfbf0b587dcd9e 2024-11-14T00:29:52,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8c258ad32d3a4778a2cfbf0b587dcd9e, entries=150, sequenceid=18, filesize=30.2 K 2024-11-14T00:29:52,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/fc9ab799c11145a995265e75b2737323 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/fc9ab799c11145a995265e75b2737323 2024-11-14T00:29:52,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T00:29:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:52,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T00:29:52,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/fc9ab799c11145a995265e75b2737323, entries=150, sequenceid=18, filesize=11.7 K 2024-11-14T00:29:52,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/5bab2d396e5b4597bc2fb3d5fe415994 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/5bab2d396e5b4597bc2fb3d5fe415994 2024-11-14T00:29:52,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/5bab2d396e5b4597bc2fb3d5fe415994, entries=150, sequenceid=18, filesize=11.7 K 2024-11-14T00:29:52,844 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-14T00:29:52,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 4779df6820910ec85365292104f48afc in 1136ms, sequenceid=18, compaction requested=false 2024-11-14T00:29:52,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:29:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:29:52,881 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:52,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-14T00:29:52,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111418feeda1f153424ebf0dac286ca0e492_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544191735/Put/seqid=0 2024-11-14T00:29:52,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:52,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544252902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:52,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544252904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:52,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544252908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:52,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544252908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:52,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:52,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544252908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:52,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:52,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742334_1510 (size=12154) 2024-11-14T00:29:52,937 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:52,965 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111418feeda1f153424ebf0dac286ca0e492_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111418feeda1f153424ebf0dac286ca0e492_4779df6820910ec85365292104f48afc 2024-11-14T00:29:52,971 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/42b1f8a8981941bc992a9680884c0b51, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:29:52,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/42b1f8a8981941bc992a9680884c0b51 is 175, key is test_row_0/A:col10/1731544191735/Put/seqid=0 2024-11-14T00:29:52,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T00:29:52,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:52,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:52,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:52,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742335_1511 (size=30955) 2024-11-14T00:29:53,012 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/42b1f8a8981941bc992a9680884c0b51 2024-11-14T00:29:53,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:53,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544253020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:53,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:53,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:53,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544253025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:53,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:53,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544253025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:53,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544253025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:53,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:53,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:53,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544253025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:53,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:53,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/9137700e0c4a41a2a29f3a5670b21925 is 50, key is test_row_0/B:col10/1731544191735/Put/seqid=0 2024-11-14T00:29:53,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742336_1512 (size=12001) 2024-11-14T00:29:53,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T00:29:53,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:53,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:53,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:53,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,207 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T00:29:53,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:53,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544253237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:53,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544253237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:53,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:53,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544253237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:53,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:53,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544253237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:53,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:53,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:53,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544253240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:53,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:53,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T00:29:53,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:53,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:53,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:53,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T00:29:53,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:53,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:53,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:53,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/9137700e0c4a41a2a29f3a5670b21925 2024-11-14T00:29:53,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/be215ea3ce9c4d15b15481ffd9ac4211 is 50, key is test_row_0/C:col10/1731544191735/Put/seqid=0 2024-11-14T00:29:53,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:53,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544253546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:53,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544253547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:53,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544253555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:53,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544253556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:53,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:53,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544253556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:53,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:53,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742337_1513 (size=12001) 2024-11-14T00:29:53,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T00:29:53,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:53,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:53,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:53,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T00:29:53,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:53,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:53,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:53,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T00:29:53,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T00:29:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:53,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:53,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:53,918 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T00:29:53,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:54,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/be215ea3ce9c4d15b15481ffd9ac4211 2024-11-14T00:29:54,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/42b1f8a8981941bc992a9680884c0b51 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/42b1f8a8981941bc992a9680884c0b51 2024-11-14T00:29:54,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/42b1f8a8981941bc992a9680884c0b51, entries=150, sequenceid=42, filesize=30.2 K 2024-11-14T00:29:54,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/9137700e0c4a41a2a29f3a5670b21925 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/9137700e0c4a41a2a29f3a5670b21925 2024-11-14T00:29:54,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:54,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544254056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:54,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:54,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544254065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:54,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:54,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544254065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:54,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544254065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:54,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:54,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544254068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:54,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T00:29:54,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/9137700e0c4a41a2a29f3a5670b21925, entries=150, sequenceid=42, filesize=11.7 K 2024-11-14T00:29:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:54,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:54,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:54,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/be215ea3ce9c4d15b15481ffd9ac4211 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/be215ea3ce9c4d15b15481ffd9ac4211 2024-11-14T00:29:54,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/be215ea3ce9c4d15b15481ffd9ac4211, entries=150, sequenceid=42, filesize=11.7 K 2024-11-14T00:29:54,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4779df6820910ec85365292104f48afc in 1211ms, sequenceid=42, compaction requested=false 2024-11-14T00:29:54,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:29:54,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-14T00:29:54,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:54,235 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-14T00:29:54,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111446c7738b4b174f4088e10a5d2bafe844_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544192902/Put/seqid=0 2024-11-14T00:29:54,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742338_1514 (size=12154) 2024-11-14T00:29:54,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,651 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111446c7738b4b174f4088e10a5d2bafe844_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111446c7738b4b174f4088e10a5d2bafe844_4779df6820910ec85365292104f48afc 2024-11-14T00:29:54,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/02b86f8fb41c4c28aa60a5dde8c3ab2d, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:29:54,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/02b86f8fb41c4c28aa60a5dde8c3ab2d is 175, key is test_row_0/A:col10/1731544192902/Put/seqid=0 2024-11-14T00:29:54,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742339_1515 (size=30955) 2024-11-14T00:29:54,670 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/02b86f8fb41c4c28aa60a5dde8c3ab2d 2024-11-14T00:29:54,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/d6cb1e7aef4b4ddabf5dd7e75f4b6cae is 50, key is test_row_0/B:col10/1731544192902/Put/seqid=0 2024-11-14T00:29:54,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742340_1516 (size=12001) 2024-11-14T00:29:54,709 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/d6cb1e7aef4b4ddabf5dd7e75f4b6cae 2024-11-14T00:29:54,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/a96954346e9e49e691ba0b0d0dcb658d is 50, key is test_row_0/C:col10/1731544192902/Put/seqid=0 2024-11-14T00:29:54,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742341_1517 (size=12001) 2024-11-14T00:29:54,745 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/a96954346e9e49e691ba0b0d0dcb658d 2024-11-14T00:29:54,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/02b86f8fb41c4c28aa60a5dde8c3ab2d as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/02b86f8fb41c4c28aa60a5dde8c3ab2d 2024-11-14T00:29:54,759 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/02b86f8fb41c4c28aa60a5dde8c3ab2d, entries=150, sequenceid=54, filesize=30.2 K 2024-11-14T00:29:54,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/d6cb1e7aef4b4ddabf5dd7e75f4b6cae as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/d6cb1e7aef4b4ddabf5dd7e75f4b6cae 2024-11-14T00:29:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,764 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/d6cb1e7aef4b4ddabf5dd7e75f4b6cae, entries=150, sequenceid=54, filesize=11.7 K 2024-11-14T00:29:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/a96954346e9e49e691ba0b0d0dcb658d as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/a96954346e9e49e691ba0b0d0dcb658d 2024-11-14T00:29:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,777 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/a96954346e9e49e691ba0b0d0dcb658d, entries=150, sequenceid=54, filesize=11.7 K 2024-11-14T00:29:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,778 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 4779df6820910ec85365292104f48afc in 543ms, sequenceid=54, compaction requested=true 2024-11-14T00:29:54,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:29:54,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:54,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-14T00:29:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-14T00:29:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-14T00:29:54,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0740 sec 2024-11-14T00:29:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 3.0860 sec 2024-11-14T00:29:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,134 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T00:29:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:29:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142e67f817a3324eb1abe7b78c00627c3e_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544195102/Put/seqid=0 2024-11-14T00:29:55,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742342_1518 (size=19474) 2024-11-14T00:29:55,172 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:55,179 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142e67f817a3324eb1abe7b78c00627c3e_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142e67f817a3324eb1abe7b78c00627c3e_4779df6820910ec85365292104f48afc 2024-11-14T00:29:55,181 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/fac400b433f74d89915ac66ce78ffb95, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:29:55,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/fac400b433f74d89915ac66ce78ffb95 is 175, key is test_row_0/A:col10/1731544195102/Put/seqid=0 2024-11-14T00:29:55,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742343_1519 (size=56733) 2024-11-14T00:29:55,197 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=66, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/fac400b433f74d89915ac66ce78ffb95 2024-11-14T00:29:55,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/b0f9754c9a22463ca49f39927c101581 is 50, key is test_row_0/B:col10/1731544195102/Put/seqid=0 2024-11-14T00:29:55,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742344_1520 (size=12001) 2024-11-14T00:29:55,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544255276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/b0f9754c9a22463ca49f39927c101581 2024-11-14T00:29:55,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544255277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544255278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544255283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544255288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/351938c3193b4cb482d10efcea079c18 is 50, key is test_row_0/C:col10/1731544195102/Put/seqid=0 2024-11-14T00:29:55,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742345_1521 (size=12001) 2024-11-14T00:29:55,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544255395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/351938c3193b4cb482d10efcea079c18 2024-11-14T00:29:55,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544255409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544255409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544255414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544255415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/fac400b433f74d89915ac66ce78ffb95 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/fac400b433f74d89915ac66ce78ffb95 2024-11-14T00:29:55,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/fac400b433f74d89915ac66ce78ffb95, entries=300, sequenceid=66, filesize=55.4 K 2024-11-14T00:29:55,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/b0f9754c9a22463ca49f39927c101581 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/b0f9754c9a22463ca49f39927c101581 2024-11-14T00:29:55,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/b0f9754c9a22463ca49f39927c101581, entries=150, sequenceid=66, filesize=11.7 K 2024-11-14T00:29:55,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544255606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/351938c3193b4cb482d10efcea079c18 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/351938c3193b4cb482d10efcea079c18 2024-11-14T00:29:55,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544255625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544255628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544255635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544255639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/351938c3193b4cb482d10efcea079c18, entries=150, sequenceid=66, filesize=11.7 K 2024-11-14T00:29:55,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4779df6820910ec85365292104f48afc in 526ms, sequenceid=66, compaction requested=true 2024-11-14T00:29:55,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:29:55,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:55,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:55,660 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:29:55,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:55,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:55,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:55,661 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:29:55,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:55,672 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:29:55,672 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/A is initiating minor compaction (all files) 2024-11-14T00:29:55,672 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:29:55,672 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/B is initiating minor compaction (all files) 2024-11-14T00:29:55,672 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/A in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:55,672 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/B in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:55,672 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/fc9ab799c11145a995265e75b2737323, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/9137700e0c4a41a2a29f3a5670b21925, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/d6cb1e7aef4b4ddabf5dd7e75f4b6cae, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/b0f9754c9a22463ca49f39927c101581] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=46.9 K 2024-11-14T00:29:55,672 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8c258ad32d3a4778a2cfbf0b587dcd9e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/42b1f8a8981941bc992a9680884c0b51, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/02b86f8fb41c4c28aa60a5dde8c3ab2d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/fac400b433f74d89915ac66ce78ffb95] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=146.1 K 2024-11-14T00:29:55,672 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:55,673 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8c258ad32d3a4778a2cfbf0b587dcd9e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/42b1f8a8981941bc992a9680884c0b51, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/02b86f8fb41c4c28aa60a5dde8c3ab2d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/fac400b433f74d89915ac66ce78ffb95] 2024-11-14T00:29:55,675 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting fc9ab799c11145a995265e75b2737323, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731544191709 2024-11-14T00:29:55,675 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c258ad32d3a4778a2cfbf0b587dcd9e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731544191709 2024-11-14T00:29:55,676 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42b1f8a8981941bc992a9680884c0b51, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731544191731 2024-11-14T00:29:55,676 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 9137700e0c4a41a2a29f3a5670b21925, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731544191731 2024-11-14T00:29:55,680 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02b86f8fb41c4c28aa60a5dde8c3ab2d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731544192902 2024-11-14T00:29:55,681 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting d6cb1e7aef4b4ddabf5dd7e75f4b6cae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731544192902 2024-11-14T00:29:55,681 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting fac400b433f74d89915ac66ce78ffb95, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731544195102 2024-11-14T00:29:55,681 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting b0f9754c9a22463ca49f39927c101581, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731544195102 2024-11-14T00:29:55,693 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:29:55,698 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#B#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:55,699 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/2e6f19a014a34afc944b2a8adb1a11dd is 50, key is test_row_0/B:col10/1731544195102/Put/seqid=0 2024-11-14T00:29:55,707 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114ba6c82147ece49d89890eb01de45539b_4779df6820910ec85365292104f48afc store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:29:55,710 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114ba6c82147ece49d89890eb01de45539b_4779df6820910ec85365292104f48afc, store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:29:55,710 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ba6c82147ece49d89890eb01de45539b_4779df6820910ec85365292104f48afc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:29:55,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742346_1522 (size=12139) 2024-11-14T00:29:55,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742347_1523 (size=4469) 2024-11-14T00:29:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-14T00:29:55,883 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:29:55,888 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:29:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-14T00:29:55,891 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:29:55,892 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:29:55,892 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:29:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T00:29:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:29:55,927 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:55,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-14T00:29:55,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544255952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544255956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544255956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bda3f6eb2918427e8ec78ba2277a0ab5_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544195920/Put/seqid=0 2024-11-14T00:29:55,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544255961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544255962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:55,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742348_1524 (size=12154) 2024-11-14T00:29:55,998 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T00:29:56,010 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bda3f6eb2918427e8ec78ba2277a0ab5_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bda3f6eb2918427e8ec78ba2277a0ab5_4779df6820910ec85365292104f48afc 2024-11-14T00:29:56,016 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/f21d209fa6124ec6b28134ff1c28b17f, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:29:56,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/f21d209fa6124ec6b28134ff1c28b17f is 175, key is test_row_0/A:col10/1731544195920/Put/seqid=0 2024-11-14T00:29:56,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742349_1525 (size=30955) 2024-11-14T00:29:56,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T00:29:56,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:56,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:56,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:56,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544256066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:56,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544256075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:56,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544256075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:56,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544256075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:56,152 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/2e6f19a014a34afc944b2a8adb1a11dd as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/2e6f19a014a34afc944b2a8adb1a11dd 2024-11-14T00:29:56,159 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4779df6820910ec85365292104f48afc/B of 4779df6820910ec85365292104f48afc into 2e6f19a014a34afc944b2a8adb1a11dd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:56,159 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:29:56,159 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/B, priority=12, startTime=1731544195661; duration=0sec 2024-11-14T00:29:56,159 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:56,159 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:B 2024-11-14T00:29:56,159 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:29:56,161 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:29:56,161 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/C is initiating minor compaction (all files) 2024-11-14T00:29:56,161 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/C in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:56,161 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/5bab2d396e5b4597bc2fb3d5fe415994, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/be215ea3ce9c4d15b15481ffd9ac4211, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/a96954346e9e49e691ba0b0d0dcb658d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/351938c3193b4cb482d10efcea079c18] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=46.9 K 2024-11-14T00:29:56,161 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bab2d396e5b4597bc2fb3d5fe415994, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731544191709 2024-11-14T00:29:56,162 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting be215ea3ce9c4d15b15481ffd9ac4211, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1731544191731 2024-11-14T00:29:56,162 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting a96954346e9e49e691ba0b0d0dcb658d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731544192902 2024-11-14T00:29:56,162 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 351938c3193b4cb482d10efcea079c18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731544195102 2024-11-14T00:29:56,163 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#A#compaction#444 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:56,164 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/7b5ecb4ff2e346188f6aa0679565e473 is 175, key is test_row_0/A:col10/1731544195102/Put/seqid=0 2024-11-14T00:29:56,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742350_1526 (size=31093) 2024-11-14T00:29:56,178 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#C#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:56,178 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/8d388174faa943bd981d4f75e46d21de is 50, key is test_row_0/C:col10/1731544195102/Put/seqid=0 2024-11-14T00:29:56,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742351_1527 (size=12139) 2024-11-14T00:29:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T00:29:56,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T00:29:56,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:56,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:56,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:56,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544256275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:56,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:56,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544256285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544256285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544256285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:56,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T00:29:56,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:56,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:56,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:56,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,437 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/f21d209fa6124ec6b28134ff1c28b17f 2024-11-14T00:29:56,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/1b2110f3ceed469cae4396a2effcc90a is 50, key is test_row_0/B:col10/1731544195920/Put/seqid=0 2024-11-14T00:29:56,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742352_1528 (size=12001) 2024-11-14T00:29:56,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:56,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544256475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:56,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T00:29:56,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T00:29:56,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:56,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:56,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:56,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,583 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/7b5ecb4ff2e346188f6aa0679565e473 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/7b5ecb4ff2e346188f6aa0679565e473 2024-11-14T00:29:56,587 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4779df6820910ec85365292104f48afc/A of 4779df6820910ec85365292104f48afc into 7b5ecb4ff2e346188f6aa0679565e473(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:56,587 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:29:56,587 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/A, priority=12, startTime=1731544195660; duration=0sec 2024-11-14T00:29:56,587 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:56,587 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:A 2024-11-14T00:29:56,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544256585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:56,595 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/8d388174faa943bd981d4f75e46d21de as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/8d388174faa943bd981d4f75e46d21de 2024-11-14T00:29:56,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544256595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:56,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544256595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544256596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:56,598 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4779df6820910ec85365292104f48afc/C of 4779df6820910ec85365292104f48afc into 8d388174faa943bd981d4f75e46d21de(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:56,598 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:29:56,598 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/C, priority=12, startTime=1731544195661; duration=0sec 2024-11-14T00:29:56,598 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:56,598 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:C 2024-11-14T00:29:56,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T00:29:56,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:56,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:56,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:56,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T00:29:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:56,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/1b2110f3ceed469cae4396a2effcc90a 2024-11-14T00:29:56,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/fa07c3a984794f5094155185075df340 is 50, key is test_row_0/C:col10/1731544195920/Put/seqid=0 2024-11-14T00:29:56,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742353_1529 (size=12001) 2024-11-14T00:29:56,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T00:29:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:56,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T00:29:57,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544257095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:57,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:57,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544257105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:57,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:57,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544257105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544257106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:57,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:57,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:57,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:57,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T00:29:57,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:57,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:57,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:57,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T00:29:57,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:57,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:57,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:57,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/fa07c3a984794f5094155185075df340 2024-11-14T00:29:57,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/f21d209fa6124ec6b28134ff1c28b17f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f21d209fa6124ec6b28134ff1c28b17f 2024-11-14T00:29:57,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f21d209fa6124ec6b28134ff1c28b17f, entries=150, sequenceid=91, filesize=30.2 K 2024-11-14T00:29:57,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/1b2110f3ceed469cae4396a2effcc90a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/1b2110f3ceed469cae4396a2effcc90a 2024-11-14T00:29:57,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/1b2110f3ceed469cae4396a2effcc90a, entries=150, sequenceid=91, filesize=11.7 K 2024-11-14T00:29:57,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/fa07c3a984794f5094155185075df340 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/fa07c3a984794f5094155185075df340 2024-11-14T00:29:57,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T00:29:57,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:57,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:57,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:57,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/fa07c3a984794f5094155185075df340, entries=150, sequenceid=91, filesize=11.7 K 2024-11-14T00:29:57,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 4779df6820910ec85365292104f48afc in 1546ms, sequenceid=91, compaction requested=false 2024-11-14T00:29:57,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:29:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:29:57,492 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:57,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-14T00:29:57,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e606835734f74a21b69dbf4ab2c25f13_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544195952/Put/seqid=0 2024-11-14T00:29:57,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742354_1530 (size=12154) 2024-11-14T00:29:57,594 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:57,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T00:29:57,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:57,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:57,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:57,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,622 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e606835734f74a21b69dbf4ab2c25f13_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e606835734f74a21b69dbf4ab2c25f13_4779df6820910ec85365292104f48afc 2024-11-14T00:29:57,638 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/f56899df841d4b20927371bfa52cb30b, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:29:57,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/f56899df841d4b20927371bfa52cb30b is 175, key is test_row_0/A:col10/1731544195952/Put/seqid=0 2024-11-14T00:29:57,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544257639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:57,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:57,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742355_1531 (size=30955) 2024-11-14T00:29:57,695 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=107, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/f56899df841d4b20927371bfa52cb30b 2024-11-14T00:29:57,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/c01ab9231b364f8bb9b46b8dd7c152d7 is 50, key is test_row_0/B:col10/1731544195952/Put/seqid=0 2024-11-14T00:29:57,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:57,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544257756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:57,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:57,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:57,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:57,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T00:29:57,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:57,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:57,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:57,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742356_1532 (size=12001) 2024-11-14T00:29:57,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/c01ab9231b364f8bb9b46b8dd7c152d7 2024-11-14T00:29:57,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/f24c17edf7c8459d95cadd21f53145c6 is 50, key is test_row_0/C:col10/1731544195952/Put/seqid=0 2024-11-14T00:29:57,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742357_1533 (size=12001) 2024-11-14T00:29:57,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T00:29:57,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:57,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:57,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:57,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:57,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:57,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544257976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:58,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T00:29:58,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T00:29:58,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:58,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:58,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:58,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:58,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:58,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:58,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544258106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:58,112 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_2' in 4779df6820910ec85365292104f48afc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:58,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:58,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:58,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:58,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:58,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544258116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:58,119 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_0' in 4779df6820910ec85365292104f48afc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:58,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:58,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:58,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:58,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:58,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544258116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:58,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:58,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544258116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:58,120 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_1' in 4779df6820910ec85365292104f48afc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:58,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:58,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:58,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:58,121 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_1' in 4779df6820910ec85365292104f48afc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:29:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:58,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T00:29:58,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:58,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:58,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:58,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:58,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:29:58,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544258285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:58,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:58,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/f24c17edf7c8459d95cadd21f53145c6 2024-11-14T00:29:58,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/f56899df841d4b20927371bfa52cb30b as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f56899df841d4b20927371bfa52cb30b 2024-11-14T00:29:58,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f56899df841d4b20927371bfa52cb30b, entries=150, sequenceid=107, filesize=30.2 K 2024-11-14T00:29:58,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/c01ab9231b364f8bb9b46b8dd7c152d7 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c01ab9231b364f8bb9b46b8dd7c152d7 2024-11-14T00:29:58,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c01ab9231b364f8bb9b46b8dd7c152d7, entries=150, sequenceid=107, filesize=11.7 K 2024-11-14T00:29:58,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/f24c17edf7c8459d95cadd21f53145c6 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/f24c17edf7c8459d95cadd21f53145c6 2024-11-14T00:29:58,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/f24c17edf7c8459d95cadd21f53145c6, entries=150, sequenceid=107, filesize=11.7 K 2024-11-14T00:29:58,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 4779df6820910ec85365292104f48afc in 841ms, sequenceid=107, compaction requested=true 2024-11-14T00:29:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:29:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:29:58,333 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:29:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:29:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:58,333 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:58,334 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:58,334 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:58,334 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/A is initiating minor compaction (all files) 2024-11-14T00:29:58,334 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/B is initiating minor compaction (all files) 2024-11-14T00:29:58,334 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/A in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:58,334 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/7b5ecb4ff2e346188f6aa0679565e473, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f21d209fa6124ec6b28134ff1c28b17f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f56899df841d4b20927371bfa52cb30b] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=90.8 K 2024-11-14T00:29:58,334 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:58,334 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/7b5ecb4ff2e346188f6aa0679565e473, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f21d209fa6124ec6b28134ff1c28b17f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f56899df841d4b20927371bfa52cb30b] 2024-11-14T00:29:58,334 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/B in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:58,335 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/2e6f19a014a34afc944b2a8adb1a11dd, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/1b2110f3ceed469cae4396a2effcc90a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c01ab9231b364f8bb9b46b8dd7c152d7] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=35.3 K 2024-11-14T00:29:58,335 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b5ecb4ff2e346188f6aa0679565e473, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731544195102 2024-11-14T00:29:58,335 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e6f19a014a34afc944b2a8adb1a11dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731544195102 2024-11-14T00:29:58,335 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting f21d209fa6124ec6b28134ff1c28b17f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731544195277 2024-11-14T00:29:58,335 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b2110f3ceed469cae4396a2effcc90a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731544195277 2024-11-14T00:29:58,336 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting f56899df841d4b20927371bfa52cb30b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731544195952 2024-11-14T00:29:58,336 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c01ab9231b364f8bb9b46b8dd7c152d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731544195952 2024-11-14T00:29:58,345 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:29:58,347 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#B#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:58,348 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/c08f18db2e8b43cca5dc73dd1ad1617f is 50, key is test_row_0/B:col10/1731544195952/Put/seqid=0 2024-11-14T00:29:58,349 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411141ac43dac040845f6b5dba76c24997b03_4779df6820910ec85365292104f48afc store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:29:58,351 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411141ac43dac040845f6b5dba76c24997b03_4779df6820910ec85365292104f48afc, store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:29:58,351 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141ac43dac040845f6b5dba76c24997b03_4779df6820910ec85365292104f48afc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:29:58,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742358_1534 (size=12241) 2024-11-14T00:29:58,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742359_1535 (size=4469) 2024-11-14T00:29:58,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-14T00:29:58,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:58,417 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-14T00:29:58,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111495268acfed714a7c895c7d7104697823_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544197599/Put/seqid=0 2024-11-14T00:29:58,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742360_1536 (size=12204) 2024-11-14T00:29:58,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:58,498 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111495268acfed714a7c895c7d7104697823_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111495268acfed714a7c895c7d7104697823_4779df6820910ec85365292104f48afc 2024-11-14T00:29:58,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/734ce039ebc448c6a432b44099dc1afc, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:29:58,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/734ce039ebc448c6a432b44099dc1afc is 175, key is test_row_0/A:col10/1731544197599/Put/seqid=0 2024-11-14T00:29:58,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742361_1537 (size=31005) 2024-11-14T00:29:58,535 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/734ce039ebc448c6a432b44099dc1afc 2024-11-14T00:29:58,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/c79b577e3c514130aa5bbca6f03e99bd is 50, key is test_row_0/B:col10/1731544197599/Put/seqid=0 2024-11-14T00:29:58,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742362_1538 (size=12051) 2024-11-14T00:29:58,802 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#A#compaction#453 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:58,802 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/27c3514067144c21adee605f741bdb58 is 175, key is test_row_0/A:col10/1731544195952/Put/seqid=0 2024-11-14T00:29:58,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:29:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:29:58,810 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/c08f18db2e8b43cca5dc73dd1ad1617f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c08f18db2e8b43cca5dc73dd1ad1617f 2024-11-14T00:29:58,817 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/B of 4779df6820910ec85365292104f48afc into c08f18db2e8b43cca5dc73dd1ad1617f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:58,817 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:29:58,817 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/B, priority=13, startTime=1731544198333; duration=0sec 2024-11-14T00:29:58,817 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:29:58,817 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:B 2024-11-14T00:29:58,817 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:29:58,819 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:29:58,819 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/C is initiating minor compaction (all files) 2024-11-14T00:29:58,819 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/C in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:58,819 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/8d388174faa943bd981d4f75e46d21de, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/fa07c3a984794f5094155185075df340, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/f24c17edf7c8459d95cadd21f53145c6] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=35.3 K 2024-11-14T00:29:58,819 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d388174faa943bd981d4f75e46d21de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731544195102 2024-11-14T00:29:58,820 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting fa07c3a984794f5094155185075df340, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731544195277 2024-11-14T00:29:58,820 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting f24c17edf7c8459d95cadd21f53145c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731544195952 2024-11-14T00:29:58,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742363_1539 (size=31195) 2024-11-14T00:29:58,846 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#C#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:29:58,847 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/14f2d167de0747c89350627172f61587 is 50, key is test_row_0/C:col10/1731544195952/Put/seqid=0 2024-11-14T00:29:58,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742364_1540 (size=12241) 2024-11-14T00:29:58,908 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/14f2d167de0747c89350627172f61587 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/14f2d167de0747c89350627172f61587 2024-11-14T00:29:58,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544258927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:58,952 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/C of 4779df6820910ec85365292104f48afc into 14f2d167de0747c89350627172f61587(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:58,952 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:29:58,952 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/C, priority=13, startTime=1731544198333; duration=0sec 2024-11-14T00:29:58,953 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:58,953 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:C 2024-11-14T00:29:59,016 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/c79b577e3c514130aa5bbca6f03e99bd 2024-11-14T00:29:59,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:59,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544259045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:59,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/5bb133621e6f484dbdc0d72aed79a0d8 is 50, key is test_row_0/C:col10/1731544197599/Put/seqid=0 2024-11-14T00:29:59,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742365_1541 (size=12051) 2024-11-14T00:29:59,101 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/5bb133621e6f484dbdc0d72aed79a0d8 2024-11-14T00:29:59,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/734ce039ebc448c6a432b44099dc1afc as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/734ce039ebc448c6a432b44099dc1afc 2024-11-14T00:29:59,135 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/734ce039ebc448c6a432b44099dc1afc, entries=150, sequenceid=130, filesize=30.3 K 2024-11-14T00:29:59,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/c79b577e3c514130aa5bbca6f03e99bd as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c79b577e3c514130aa5bbca6f03e99bd 2024-11-14T00:29:59,178 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c79b577e3c514130aa5bbca6f03e99bd, entries=150, sequenceid=130, filesize=11.8 K 2024-11-14T00:29:59,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/5bb133621e6f484dbdc0d72aed79a0d8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/5bb133621e6f484dbdc0d72aed79a0d8 2024-11-14T00:29:59,203 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/5bb133621e6f484dbdc0d72aed79a0d8, entries=150, sequenceid=130, filesize=11.8 K 2024-11-14T00:29:59,204 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 4779df6820910ec85365292104f48afc in 787ms, sequenceid=130, compaction requested=false 2024-11-14T00:29:59,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:29:59,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:29:59,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-14T00:29:59,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-14T00:29:59,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-14T00:29:59,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3140 sec 2024-11-14T00:29:59,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 3.3240 sec 2024-11-14T00:29:59,239 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/27c3514067144c21adee605f741bdb58 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/27c3514067144c21adee605f741bdb58 2024-11-14T00:29:59,244 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/A of 4779df6820910ec85365292104f48afc into 27c3514067144c21adee605f741bdb58(size=30.5 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:29:59,244 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:29:59,244 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/A, priority=13, startTime=1731544198333; duration=0sec 2024-11-14T00:29:59,244 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:29:59,244 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:A 2024-11-14T00:29:59,266 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:29:59,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T00:29:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:29:59,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b8b7a6c413424d05a98f2a7e7aacdb75_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544199265/Put/seqid=0 2024-11-14T00:29:59,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742366_1542 (size=12304) 2024-11-14T00:29:59,300 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:29:59,308 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b8b7a6c413424d05a98f2a7e7aacdb75_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b8b7a6c413424d05a98f2a7e7aacdb75_4779df6820910ec85365292104f48afc 2024-11-14T00:29:59,312 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/a0619950247d4b41b6de3db031b86536, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:29:59,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/a0619950247d4b41b6de3db031b86536 is 175, key is test_row_0/A:col10/1731544199265/Put/seqid=0 2024-11-14T00:29:59,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742367_1543 (size=31105) 2024-11-14T00:29:59,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544259387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:59,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:59,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:59,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:59,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544259494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:59,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:59,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:59,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:59,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:29:59,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544259708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:29:59,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:59,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:29:59,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:29:59,741 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/a0619950247d4b41b6de3db031b86536 2024-11-14T00:29:59,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/f0e042809dc640beace0ccf2be6665fd is 50, key is test_row_0/B:col10/1731544199265/Put/seqid=0 2024-11-14T00:29:59,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742368_1544 (size=12151) 2024-11-14T00:30:00,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544260020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:00,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-14T00:30:00,056 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:30:00,056 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:30:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-14T00:30:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T00:30:00,058 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:30:00,058 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:30:00,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:30:00,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544260128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:00,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544260128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:00,131 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_2' in 4779df6820910ec85365292104f48afc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:30:00,131 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_0' in 4779df6820910ec85365292104f48afc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:30:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:00,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544260135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:00,139 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_1' in 4779df6820910ec85365292104f48afc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:30:00,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:00,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:00,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:00,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544260145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:00,150 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_1' in 4779df6820910ec85365292104f48afc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:30:00,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:00,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:00,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T00:30:00,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/f0e042809dc640beace0ccf2be6665fd 2024-11-14T00:30:00,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T00:30:00,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:00,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:00,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:00,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:00,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:00,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/43bcb3e1ea82480b864430c4e3ba53ab is 50, key is test_row_0/C:col10/1731544199265/Put/seqid=0 2024-11-14T00:30:00,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742369_1545 (size=12151) 2024-11-14T00:30:00,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/43bcb3e1ea82480b864430c4e3ba53ab 2024-11-14T00:30:00,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/a0619950247d4b41b6de3db031b86536 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/a0619950247d4b41b6de3db031b86536 2024-11-14T00:30:00,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/a0619950247d4b41b6de3db031b86536, entries=150, sequenceid=147, filesize=30.4 K 2024-11-14T00:30:00,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/f0e042809dc640beace0ccf2be6665fd as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/f0e042809dc640beace0ccf2be6665fd 2024-11-14T00:30:00,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/f0e042809dc640beace0ccf2be6665fd, entries=150, sequenceid=147, filesize=11.9 K 2024-11-14T00:30:00,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/43bcb3e1ea82480b864430c4e3ba53ab as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/43bcb3e1ea82480b864430c4e3ba53ab 2024-11-14T00:30:00,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/43bcb3e1ea82480b864430c4e3ba53ab, entries=150, sequenceid=147, filesize=11.9 K 2024-11-14T00:30:00,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 4779df6820910ec85365292104f48afc in 1034ms, sequenceid=147, compaction requested=true 2024-11-14T00:30:00,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:00,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:30:00,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:00,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:30:00,301 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:30:00,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:30:00,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:30:00,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:30:00,301 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:30:00,302 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:30:00,302 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/B is initiating minor compaction (all files) 2024-11-14T00:30:00,302 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/B in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:00,302 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c08f18db2e8b43cca5dc73dd1ad1617f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c79b577e3c514130aa5bbca6f03e99bd, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/f0e042809dc640beace0ccf2be6665fd] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=35.6 K 2024-11-14T00:30:00,305 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c08f18db2e8b43cca5dc73dd1ad1617f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731544195952 2024-11-14T00:30:00,305 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93305 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:30:00,305 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/A is initiating minor compaction (all files) 2024-11-14T00:30:00,305 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/A in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:00,305 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/27c3514067144c21adee605f741bdb58, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/734ce039ebc448c6a432b44099dc1afc, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/a0619950247d4b41b6de3db031b86536] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=91.1 K 2024-11-14T00:30:00,305 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:00,305 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/27c3514067144c21adee605f741bdb58, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/734ce039ebc448c6a432b44099dc1afc, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/a0619950247d4b41b6de3db031b86536] 2024-11-14T00:30:00,308 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c79b577e3c514130aa5bbca6f03e99bd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731544197599 2024-11-14T00:30:00,308 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27c3514067144c21adee605f741bdb58, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731544195952 2024-11-14T00:30:00,309 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 734ce039ebc448c6a432b44099dc1afc, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731544197599 2024-11-14T00:30:00,309 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0619950247d4b41b6de3db031b86536, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731544198885 2024-11-14T00:30:00,310 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting f0e042809dc640beace0ccf2be6665fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731544198885 2024-11-14T00:30:00,319 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:00,322 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114c91ca1f193c446a596b687e84acf3320_4779df6820910ec85365292104f48afc store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:00,324 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114c91ca1f193c446a596b687e84acf3320_4779df6820910ec85365292104f48afc, store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:00,325 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c91ca1f193c446a596b687e84acf3320_4779df6820910ec85365292104f48afc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:00,332 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#B#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:00,332 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/ef3b574a71614ad9a8edca261432fa49 is 50, key is test_row_0/B:col10/1731544199265/Put/seqid=0 2024-11-14T00:30:00,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742370_1546 (size=4469) 2024-11-14T00:30:00,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742371_1547 (size=12493) 2024-11-14T00:30:00,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T00:30:00,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:00,373 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T00:30:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T00:30:00,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147c5427c0fa324a06883ef8b231bb775f_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544199349/Put/seqid=0 2024-11-14T00:30:00,388 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/ef3b574a71614ad9a8edca261432fa49 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/ef3b574a71614ad9a8edca261432fa49 2024-11-14T00:30:00,399 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/B of 4779df6820910ec85365292104f48afc into ef3b574a71614ad9a8edca261432fa49(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:00,399 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:00,399 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/B, priority=13, startTime=1731544200301; duration=0sec 2024-11-14T00:30:00,399 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:30:00,400 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:B 2024-11-14T00:30:00,400 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:30:00,401 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:30:00,401 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/C is initiating minor compaction (all files) 2024-11-14T00:30:00,401 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/C in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:00,401 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/14f2d167de0747c89350627172f61587, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/5bb133621e6f484dbdc0d72aed79a0d8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/43bcb3e1ea82480b864430c4e3ba53ab] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=35.6 K 2024-11-14T00:30:00,401 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 14f2d167de0747c89350627172f61587, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731544195952 2024-11-14T00:30:00,401 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bb133621e6f484dbdc0d72aed79a0d8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731544197599 2024-11-14T00:30:00,401 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 43bcb3e1ea82480b864430c4e3ba53ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731544198885 2024-11-14T00:30:00,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742372_1548 (size=12304) 2024-11-14T00:30:00,412 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#C#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:00,412 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/157972e557674bf8a59ef50cc2f0172c is 50, key is test_row_0/C:col10/1731544199265/Put/seqid=0 2024-11-14T00:30:00,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742373_1549 (size=12493) 2024-11-14T00:30:00,431 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/157972e557674bf8a59ef50cc2f0172c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/157972e557674bf8a59ef50cc2f0172c 2024-11-14T00:30:00,459 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/C of 4779df6820910ec85365292104f48afc into 157972e557674bf8a59ef50cc2f0172c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:00,459 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:00,459 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/C, priority=13, startTime=1731544200301; duration=0sec 2024-11-14T00:30:00,460 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:00,460 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:C 2024-11-14T00:30:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:30:00,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:00,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544260627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T00:30:00,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544260735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:00,750 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#A#compaction#462 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:00,751 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/f90d72489c9b4a8cbd568ea366822623 is 175, key is test_row_0/A:col10/1731544199265/Put/seqid=0 2024-11-14T00:30:00,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742374_1550 (size=31447) 2024-11-14T00:30:00,795 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/f90d72489c9b4a8cbd568ea366822623 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f90d72489c9b4a8cbd568ea366822623 2024-11-14T00:30:00,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:00,816 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/A of 4779df6820910ec85365292104f48afc into f90d72489c9b4a8cbd568ea366822623(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:00,816 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:00,816 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/A, priority=13, startTime=1731544200300; duration=0sec 2024-11-14T00:30:00,816 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:00,816 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:A 2024-11-14T00:30:00,823 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147c5427c0fa324a06883ef8b231bb775f_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147c5427c0fa324a06883ef8b231bb775f_4779df6820910ec85365292104f48afc 2024-11-14T00:30:00,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/64191ad002934f1ab8a212b3d485be2f, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:00,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/64191ad002934f1ab8a212b3d485be2f is 175, key is test_row_0/A:col10/1731544199349/Put/seqid=0 2024-11-14T00:30:00,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742375_1551 (size=31105) 2024-11-14T00:30:00,876 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/64191ad002934f1ab8a212b3d485be2f 2024-11-14T00:30:00,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/6870907f9d3e4161968250e78f257491 is 50, key is test_row_0/B:col10/1731544199349/Put/seqid=0 2024-11-14T00:30:00,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742376_1552 (size=12151) 2024-11-14T00:30:00,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544260945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:00,957 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/6870907f9d3e4161968250e78f257491 2024-11-14T00:30:01,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/9349c4564be14c7c83662218842b3297 is 50, key is test_row_0/C:col10/1731544199349/Put/seqid=0 2024-11-14T00:30:01,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742377_1553 (size=12151) 2024-11-14T00:30:01,064 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/9349c4564be14c7c83662218842b3297 2024-11-14T00:30:01,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/64191ad002934f1ab8a212b3d485be2f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/64191ad002934f1ab8a212b3d485be2f 2024-11-14T00:30:01,098 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/64191ad002934f1ab8a212b3d485be2f, entries=150, sequenceid=169, filesize=30.4 K 2024-11-14T00:30:01,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/6870907f9d3e4161968250e78f257491 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/6870907f9d3e4161968250e78f257491 2024-11-14T00:30:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,102 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/6870907f9d3e4161968250e78f257491, entries=150, sequenceid=169, filesize=11.9 K 2024-11-14T00:30:01,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/9349c4564be14c7c83662218842b3297 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/9349c4564be14c7c83662218842b3297 2024-11-14T00:30:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,108 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/9349c4564be14c7c83662218842b3297, entries=150, sequenceid=169, filesize=11.9 K 2024-11-14T00:30:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,109 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 4779df6820910ec85365292104f48afc in 736ms, sequenceid=169, compaction requested=false 2024-11-14T00:30:01,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:01,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:01,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-14T00:30:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-14T00:30:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-14T00:30:01,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0550 sec 2024-11-14T00:30:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.0630 sec 2024-11-14T00:30:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T00:30:01,195 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:30:01,195 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:30:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-14T00:30:01,197 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:30:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T00:30:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,198 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:30:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:30:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:30:01,265 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:30:01,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T00:30:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114eb3bb008c2b34f418ce0f74417a64b1c_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544201262/Put/seqid=0 2024-11-14T00:30:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T00:30:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742378_1554 (size=12304) 2024-11-14T00:30:01,330 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:01,335 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114eb3bb008c2b34f418ce0f74417a64b1c_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb3bb008c2b34f418ce0f74417a64b1c_4779df6820910ec85365292104f48afc 2024-11-14T00:30:01,336 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/0bcf27a4635a4bb59f18aeec64da9652, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:01,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/0bcf27a4635a4bb59f18aeec64da9652 is 175, key is test_row_0/A:col10/1731544201262/Put/seqid=0 2024-11-14T00:30:01,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742379_1555 (size=31105) 2024-11-14T00:30:01,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T00:30:01,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:01,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:01,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:01,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:01,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:01,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544261406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:01,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T00:30:01,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:01,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:01,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:01,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:01,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T00:30:01,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544261515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:01,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:01,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:01,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:01,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T00:30:01,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:01,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:01,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:01,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:01,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:01,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544261736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:01,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:01,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:01,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:01,752 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/0bcf27a4635a4bb59f18aeec64da9652 2024-11-14T00:30:01,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/adc624618b1c42c8b4c52cc4dcc9575c is 50, key is test_row_0/B:col10/1731544201262/Put/seqid=0 2024-11-14T00:30:01,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742380_1556 (size=12151) 2024-11-14T00:30:01,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/adc624618b1c42c8b4c52cc4dcc9575c 2024-11-14T00:30:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T00:30:01,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/d03a720a34e644babb9c93eda8e2690e is 50, key is test_row_0/C:col10/1731544201262/Put/seqid=0 2024-11-14T00:30:01,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T00:30:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:01,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:01,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:01,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:01,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:01,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742381_1557 (size=12151) 2024-11-14T00:30:02,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T00:30:02,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:02,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:02,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:02,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:02,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:02,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544262045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:02,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:02,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T00:30:02,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:02,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:02,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:02,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:02,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:02,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/d03a720a34e644babb9c93eda8e2690e 2024-11-14T00:30:02,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/0bcf27a4635a4bb59f18aeec64da9652 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/0bcf27a4635a4bb59f18aeec64da9652 2024-11-14T00:30:02,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/0bcf27a4635a4bb59f18aeec64da9652, entries=150, sequenceid=187, filesize=30.4 K 2024-11-14T00:30:02,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/adc624618b1c42c8b4c52cc4dcc9575c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/adc624618b1c42c8b4c52cc4dcc9575c 2024-11-14T00:30:02,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/adc624618b1c42c8b4c52cc4dcc9575c, entries=150, sequenceid=187, filesize=11.9 K 2024-11-14T00:30:02,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/d03a720a34e644babb9c93eda8e2690e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/d03a720a34e644babb9c93eda8e2690e 2024-11-14T00:30:02,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/d03a720a34e644babb9c93eda8e2690e, entries=150, sequenceid=187, filesize=11.9 K 2024-11-14T00:30:02,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 4779df6820910ec85365292104f48afc in 1046ms, sequenceid=187, compaction requested=true 2024-11-14T00:30:02,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:02,311 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:30:02,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:30:02,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:02,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:30:02,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:02,311 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:30:02,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:30:02,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:30:02,313 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:30:02,314 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/B is initiating minor compaction (all files) 2024-11-14T00:30:02,314 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/B in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:02,314 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/ef3b574a71614ad9a8edca261432fa49, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/6870907f9d3e4161968250e78f257491, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/adc624618b1c42c8b4c52cc4dcc9575c] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=35.9 K 2024-11-14T00:30:02,315 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting ef3b574a71614ad9a8edca261432fa49, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731544198885 2024-11-14T00:30:02,316 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 6870907f9d3e4161968250e78f257491, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731544199349 2024-11-14T00:30:02,316 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting adc624618b1c42c8b4c52cc4dcc9575c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731544200614 2024-11-14T00:30:02,321 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:30:02,321 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/A is initiating minor compaction (all files) 2024-11-14T00:30:02,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T00:30:02,321 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/A in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:02,321 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f90d72489c9b4a8cbd568ea366822623, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/64191ad002934f1ab8a212b3d485be2f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/0bcf27a4635a4bb59f18aeec64da9652] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=91.5 K 2024-11-14T00:30:02,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:02,321 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:02,321 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f90d72489c9b4a8cbd568ea366822623, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/64191ad002934f1ab8a212b3d485be2f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/0bcf27a4635a4bb59f18aeec64da9652] 2024-11-14T00:30:02,321 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T00:30:02,323 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting f90d72489c9b4a8cbd568ea366822623, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731544198885 2024-11-14T00:30:02,324 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64191ad002934f1ab8a212b3d485be2f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731544199349 2024-11-14T00:30:02,325 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bcf27a4635a4bb59f18aeec64da9652, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731544200614 2024-11-14T00:30:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T00:30:02,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bab61fc7b4494f05948ba1b25b82e823_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544201390/Put/seqid=0 2024-11-14T00:30:02,369 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#B#compaction#472 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:02,369 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/9421a05a2384418baa178f176d5efba2 is 50, key is test_row_0/B:col10/1731544201262/Put/seqid=0 2024-11-14T00:30:02,378 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:02,387 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114d3d196f86a434efb987124f04347152e_4779df6820910ec85365292104f48afc store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:02,390 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114d3d196f86a434efb987124f04347152e_4779df6820910ec85365292104f48afc, store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:02,390 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d3d196f86a434efb987124f04347152e_4779df6820910ec85365292104f48afc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:02,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742382_1558 (size=12595) 2024-11-14T00:30:02,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742383_1559 (size=12304) 2024-11-14T00:30:02,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742384_1560 (size=4469) 2024-11-14T00:30:02,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:30:02,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:02,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544262646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:02,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:02,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:02,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:02,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:02,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544262760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:02,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:02,833 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bab61fc7b4494f05948ba1b25b82e823_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bab61fc7b4494f05948ba1b25b82e823_4779df6820910ec85365292104f48afc 2024-11-14T00:30:02,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/3a61cc99ee2b42c395ec94353dca864a, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:02,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/3a61cc99ee2b42c395ec94353dca864a is 175, key is test_row_0/A:col10/1731544201390/Put/seqid=0 2024-11-14T00:30:02,840 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/9421a05a2384418baa178f176d5efba2 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/9421a05a2384418baa178f176d5efba2 2024-11-14T00:30:02,852 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/B of 4779df6820910ec85365292104f48afc into 9421a05a2384418baa178f176d5efba2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:02,852 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:02,853 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/B, priority=13, startTime=1731544202311; duration=0sec 2024-11-14T00:30:02,853 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:30:02,853 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:B 2024-11-14T00:30:02,853 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:30:02,856 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#A#compaction#473 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:02,856 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/84df7f6252df4037b8c8d08b23ad3c07 is 175, key is test_row_0/A:col10/1731544201262/Put/seqid=0 2024-11-14T00:30:02,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742385_1561 (size=31105) 2024-11-14T00:30:02,868 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:30:02,868 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/C is initiating minor compaction (all files) 2024-11-14T00:30:02,869 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/C in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:02,869 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/157972e557674bf8a59ef50cc2f0172c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/9349c4564be14c7c83662218842b3297, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/d03a720a34e644babb9c93eda8e2690e] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=35.9 K 2024-11-14T00:30:02,872 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 157972e557674bf8a59ef50cc2f0172c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731544198885 2024-11-14T00:30:02,877 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 9349c4564be14c7c83662218842b3297, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731544199349 2024-11-14T00:30:02,883 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting d03a720a34e644babb9c93eda8e2690e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731544200614 2024-11-14T00:30:02,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742386_1562 (size=31549) 2024-11-14T00:30:02,915 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#C#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:02,915 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/0ab73e832ef44e6db24b5f6249f8d3e4 is 50, key is test_row_0/C:col10/1731544201262/Put/seqid=0 2024-11-14T00:30:02,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742387_1563 (size=12595) 2024-11-14T00:30:02,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544262975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:02,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:03,261 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/3a61cc99ee2b42c395ec94353dca864a 2024-11-14T00:30:03,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544263288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:03,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/1ea81481eb78409d8a79d7bb6fba4658 is 50, key is test_row_0/B:col10/1731544201390/Put/seqid=0 2024-11-14T00:30:03,348 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/84df7f6252df4037b8c8d08b23ad3c07 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/84df7f6252df4037b8c8d08b23ad3c07 2024-11-14T00:30:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T00:30:03,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742388_1564 (size=12151) 2024-11-14T00:30:03,398 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/1ea81481eb78409d8a79d7bb6fba4658 2024-11-14T00:30:03,404 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/A of 4779df6820910ec85365292104f48afc into 84df7f6252df4037b8c8d08b23ad3c07(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-14T00:30:03,404 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:03,404 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/A, priority=13, startTime=1731544202311; duration=1sec 2024-11-14T00:30:03,405 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:03,405 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:A 2024-11-14T00:30:03,409 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/0ab73e832ef44e6db24b5f6249f8d3e4 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/0ab73e832ef44e6db24b5f6249f8d3e4 2024-11-14T00:30:03,422 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/C of 4779df6820910ec85365292104f48afc into 0ab73e832ef44e6db24b5f6249f8d3e4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:03,422 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:03,423 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/C, priority=13, startTime=1731544202311; duration=0sec 2024-11-14T00:30:03,423 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:03,423 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:C 2024-11-14T00:30:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/9f0fca09f0314beb88e85225face8c45 is 50, key is test_row_0/C:col10/1731544201390/Put/seqid=0 2024-11-14T00:30:03,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742389_1565 (size=12151) 2024-11-14T00:30:03,458 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/9f0fca09f0314beb88e85225face8c45 2024-11-14T00:30:03,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/3a61cc99ee2b42c395ec94353dca864a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/3a61cc99ee2b42c395ec94353dca864a 2024-11-14T00:30:03,470 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/3a61cc99ee2b42c395ec94353dca864a, entries=150, sequenceid=208, filesize=30.4 K 2024-11-14T00:30:03,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/1ea81481eb78409d8a79d7bb6fba4658 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/1ea81481eb78409d8a79d7bb6fba4658 2024-11-14T00:30:03,484 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/1ea81481eb78409d8a79d7bb6fba4658, entries=150, sequenceid=208, filesize=11.9 K 2024-11-14T00:30:03,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/9f0fca09f0314beb88e85225face8c45 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/9f0fca09f0314beb88e85225face8c45 2024-11-14T00:30:03,490 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/9f0fca09f0314beb88e85225face8c45, entries=150, sequenceid=208, filesize=11.9 K 2024-11-14T00:30:03,496 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 4779df6820910ec85365292104f48afc in 1175ms, sequenceid=208, compaction requested=false 2024-11-14T00:30:03,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:03,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:03,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-14T00:30:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-14T00:30:03,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-14T00:30:03,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3050 sec 2024-11-14T00:30:03,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.3140 sec 2024-11-14T00:30:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:30:03,817 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:30:03,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-14T00:30:03,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fe6bc01e1826493ab702b44837154381_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544202598/Put/seqid=0 2024-11-14T00:30:03,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742390_1566 (size=12304) 2024-11-14T00:30:03,901 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:03,929 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114fe6bc01e1826493ab702b44837154381_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fe6bc01e1826493ab702b44837154381_4779df6820910ec85365292104f48afc 2024-11-14T00:30:03,939 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/9ffcf9fd87fe45d6ae3dbf84e42dddd8, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:03,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/9ffcf9fd87fe45d6ae3dbf84e42dddd8 is 175, key is test_row_0/A:col10/1731544202598/Put/seqid=0 2024-11-14T00:30:03,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742391_1567 (size=31105) 2024-11-14T00:30:03,972 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/9ffcf9fd87fe45d6ae3dbf84e42dddd8 2024-11-14T00:30:04,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/484150d8aad443e48e6cde9d5559e9f7 is 50, key is test_row_0/B:col10/1731544202598/Put/seqid=0 2024-11-14T00:30:04,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742392_1568 (size=12151) 2024-11-14T00:30:04,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/484150d8aad443e48e6cde9d5559e9f7 2024-11-14T00:30:04,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:04,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544264063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:04,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:04,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/db6518564c074c578fb134e2e36873db is 50, key is test_row_0/C:col10/1731544202598/Put/seqid=0 2024-11-14T00:30:04,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742393_1569 (size=12151) 2024-11-14T00:30:04,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/db6518564c074c578fb134e2e36873db 2024-11-14T00:30:04,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60596 deadline: 1731544264157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:04,163 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_2' in 4779df6820910ec85365292104f48afc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:30:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:04,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60594 deadline: 1731544264166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:04,172 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_0' in 4779df6820910ec85365292104f48afc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:30:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:04,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/9ffcf9fd87fe45d6ae3dbf84e42dddd8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/9ffcf9fd87fe45d6ae3dbf84e42dddd8 2024-11-14T00:30:04,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60610 deadline: 1731544264175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:04,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60618 deadline: 1731544264176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:04,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544264176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:04,185 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_1' in 4779df6820910ec85365292104f48afc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:30:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:04,186 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 63baa11b75f3,33199,1731544045387 for 'test_row_1' in 4779df6820910ec85365292104f48afc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T00:30:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:04,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/9ffcf9fd87fe45d6ae3dbf84e42dddd8, entries=150, sequenceid=227, filesize=30.4 K 2024-11-14T00:30:04,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/484150d8aad443e48e6cde9d5559e9f7 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/484150d8aad443e48e6cde9d5559e9f7 2024-11-14T00:30:04,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/484150d8aad443e48e6cde9d5559e9f7, entries=150, sequenceid=227, filesize=11.9 K 2024-11-14T00:30:04,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/db6518564c074c578fb134e2e36873db as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/db6518564c074c578fb134e2e36873db 2024-11-14T00:30:04,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/db6518564c074c578fb134e2e36873db, entries=150, sequenceid=227, filesize=11.9 K 2024-11-14T00:30:04,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 4779df6820910ec85365292104f48afc in 476ms, sequenceid=227, compaction requested=true 2024-11-14T00:30:04,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:04,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:30:04,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:04,293 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:30:04,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:30:04,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:30:04,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:30:04,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:30:04,293 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:30:04,304 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:30:04,304 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/A is initiating minor compaction (all files) 2024-11-14T00:30:04,304 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/A in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:04,305 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/84df7f6252df4037b8c8d08b23ad3c07, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/3a61cc99ee2b42c395ec94353dca864a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/9ffcf9fd87fe45d6ae3dbf84e42dddd8] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=91.6 K 2024-11-14T00:30:04,305 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:04,305 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/84df7f6252df4037b8c8d08b23ad3c07, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/3a61cc99ee2b42c395ec94353dca864a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/9ffcf9fd87fe45d6ae3dbf84e42dddd8] 2024-11-14T00:30:04,308 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:30:04,308 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/B is initiating minor compaction (all files) 2024-11-14T00:30:04,308 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/B in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:04,308 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/9421a05a2384418baa178f176d5efba2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/1ea81481eb78409d8a79d7bb6fba4658, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/484150d8aad443e48e6cde9d5559e9f7] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=36.0 K 2024-11-14T00:30:04,309 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 84df7f6252df4037b8c8d08b23ad3c07, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731544200614 2024-11-14T00:30:04,310 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9421a05a2384418baa178f176d5efba2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731544200614 2024-11-14T00:30:04,312 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a61cc99ee2b42c395ec94353dca864a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731544201390 2024-11-14T00:30:04,312 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ea81481eb78409d8a79d7bb6fba4658, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731544201390 2024-11-14T00:30:04,314 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 484150d8aad443e48e6cde9d5559e9f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731544202598 2024-11-14T00:30:04,314 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ffcf9fd87fe45d6ae3dbf84e42dddd8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731544202598 2024-11-14T00:30:04,366 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#B#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:04,367 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/a88ec517bdf24ba690b5dd1dc249b6f3 is 50, key is test_row_0/B:col10/1731544202598/Put/seqid=0 2024-11-14T00:30:04,369 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:04,397 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411142a4de917c9774aa1a0bf5c82876f871c_4779df6820910ec85365292104f48afc store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:04,399 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411142a4de917c9774aa1a0bf5c82876f871c_4779df6820910ec85365292104f48afc, store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:04,399 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142a4de917c9774aa1a0bf5c82876f871c_4779df6820910ec85365292104f48afc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:04,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742394_1570 (size=12697) 2024-11-14T00:30:04,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:30:04,408 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:30:04,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-14T00:30:04,420 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/a88ec517bdf24ba690b5dd1dc249b6f3 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/a88ec517bdf24ba690b5dd1dc249b6f3 2024-11-14T00:30:04,425 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/B of 4779df6820910ec85365292104f48afc into a88ec517bdf24ba690b5dd1dc249b6f3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:04,425 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:04,425 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/B, priority=13, startTime=1731544204293; duration=0sec 2024-11-14T00:30:04,425 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:30:04,426 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:B 2024-11-14T00:30:04,426 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:30:04,431 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:30:04,431 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/C is initiating minor compaction (all files) 2024-11-14T00:30:04,431 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/C in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:04,431 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/0ab73e832ef44e6db24b5f6249f8d3e4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/9f0fca09f0314beb88e85225face8c45, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/db6518564c074c578fb134e2e36873db] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=36.0 K 2024-11-14T00:30:04,433 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ab73e832ef44e6db24b5f6249f8d3e4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731544200614 2024-11-14T00:30:04,434 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f0fca09f0314beb88e85225face8c45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731544201390 2024-11-14T00:30:04,439 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting db6518564c074c578fb134e2e36873db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731544202598 2024-11-14T00:30:04,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141768129383f148e48d213df20f82b8a1_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544203957/Put/seqid=0 2024-11-14T00:30:04,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742395_1571 (size=4469) 2024-11-14T00:30:04,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742396_1572 (size=14794) 2024-11-14T00:30:04,494 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#C#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:04,494 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/3dda014fe80c4ebdb1122048d646f047 is 50, key is test_row_0/C:col10/1731544202598/Put/seqid=0 2024-11-14T00:30:04,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742397_1573 (size=12697) 2024-11-14T00:30:04,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544264542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:04,555 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/3dda014fe80c4ebdb1122048d646f047 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/3dda014fe80c4ebdb1122048d646f047 2024-11-14T00:30:04,565 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/C of 4779df6820910ec85365292104f48afc into 3dda014fe80c4ebdb1122048d646f047(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:04,565 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:04,565 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/C, priority=13, startTime=1731544204293; duration=0sec 2024-11-14T00:30:04,565 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:04,565 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:C 2024-11-14T00:30:04,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544264655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:04,866 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#A#compaction#481 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:04,867 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/b2d7bf831f974fb0b52f536697d4144e is 175, key is test_row_0/A:col10/1731544202598/Put/seqid=0 2024-11-14T00:30:04,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:04,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544264875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:04,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:04,888 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:04,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742398_1574 (size=31651) 2024-11-14T00:30:04,911 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141768129383f148e48d213df20f82b8a1_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141768129383f148e48d213df20f82b8a1_4779df6820910ec85365292104f48afc 2024-11-14T00:30:04,915 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/c18b080dee90443da1bff516ea7b71c8, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:04,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/c18b080dee90443da1bff516ea7b71c8 is 175, key is test_row_0/A:col10/1731544203957/Put/seqid=0 2024-11-14T00:30:04,952 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/b2d7bf831f974fb0b52f536697d4144e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/b2d7bf831f974fb0b52f536697d4144e 2024-11-14T00:30:04,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742399_1575 (size=39749) 2024-11-14T00:30:04,982 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/A of 4779df6820910ec85365292104f48afc into b2d7bf831f974fb0b52f536697d4144e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:04,982 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:04,983 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/A, priority=13, startTime=1731544204293; duration=0sec 2024-11-14T00:30:04,983 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:04,983 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:A 2024-11-14T00:30:05,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544265185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:05,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:05,372 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/c18b080dee90443da1bff516ea7b71c8 2024-11-14T00:30:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T00:30:05,395 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:30:05,396 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:30:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-14T00:30:05,398 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:30:05,399 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:30:05,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:30:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T00:30:05,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/40cfc78ecc814e1da3efc9bc449cdc55 is 50, key is test_row_0/B:col10/1731544203957/Put/seqid=0 2024-11-14T00:30:05,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742400_1576 (size=12151) 2024-11-14T00:30:05,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T00:30:05,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/40cfc78ecc814e1da3efc9bc449cdc55 2024-11-14T00:30:05,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/cbc835240684494da59a8f2882318987 is 50, key is test_row_0/C:col10/1731544203957/Put/seqid=0 2024-11-14T00:30:05,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T00:30:05,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:05,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:05,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:05,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:05,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:05,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742401_1577 (size=12151) 2024-11-14T00:30:05,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544265695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:05,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T00:30:05,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:05,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:05,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:05,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:05,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T00:30:05,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T00:30:05,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:05,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:05,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:05,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:05,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:06,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/cbc835240684494da59a8f2882318987 2024-11-14T00:30:06,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/c18b080dee90443da1bff516ea7b71c8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/c18b080dee90443da1bff516ea7b71c8 2024-11-14T00:30:06,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/c18b080dee90443da1bff516ea7b71c8, entries=200, sequenceid=248, filesize=38.8 K 2024-11-14T00:30:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T00:30:06,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/40cfc78ecc814e1da3efc9bc449cdc55 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/40cfc78ecc814e1da3efc9bc449cdc55 2024-11-14T00:30:06,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T00:30:06,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:06,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:06,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:06,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:06,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:06,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/40cfc78ecc814e1da3efc9bc449cdc55, entries=150, sequenceid=248, filesize=11.9 K 2024-11-14T00:30:06,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/cbc835240684494da59a8f2882318987 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/cbc835240684494da59a8f2882318987 2024-11-14T00:30:06,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/cbc835240684494da59a8f2882318987, entries=150, sequenceid=248, filesize=11.9 K 2024-11-14T00:30:06,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 4779df6820910ec85365292104f48afc in 1644ms, sequenceid=248, compaction requested=false 2024-11-14T00:30:06,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:06,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T00:30:06,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:06,197 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-14T00:30:06,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149b7d90c91a794d658da2eacc921742d8_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544204520/Put/seqid=0 2024-11-14T00:30:06,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742402_1578 (size=12454) 2024-11-14T00:30:06,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:06,247 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149b7d90c91a794d658da2eacc921742d8_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149b7d90c91a794d658da2eacc921742d8_4779df6820910ec85365292104f48afc 2024-11-14T00:30:06,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/7f3d95f337a1450ebf8fce488542b903, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:06,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/7f3d95f337a1450ebf8fce488542b903 is 175, key is test_row_0/A:col10/1731544204520/Put/seqid=0 2024-11-14T00:30:06,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742403_1579 (size=31255) 2024-11-14T00:30:06,268 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/7f3d95f337a1450ebf8fce488542b903 2024-11-14T00:30:06,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/aceb86f3738b4153b57a363c33ae4cea is 50, key is test_row_0/B:col10/1731544204520/Put/seqid=0 2024-11-14T00:30:06,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742404_1580 (size=12301) 2024-11-14T00:30:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T00:30:06,701 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/aceb86f3738b4153b57a363c33ae4cea 2024-11-14T00:30:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:30:06,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:06,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/503b5a199ad14dd9b572ea4f68c380bc is 50, key is test_row_0/C:col10/1731544204520/Put/seqid=0 2024-11-14T00:30:06,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742405_1581 (size=12301) 2024-11-14T00:30:06,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:06,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544266821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:06,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:06,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:06,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:06,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544266936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:06,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:06,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:06,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:07,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:07,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544267148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:07,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:07,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:07,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:07,204 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/503b5a199ad14dd9b572ea4f68c380bc 2024-11-14T00:30:07,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/7f3d95f337a1450ebf8fce488542b903 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/7f3d95f337a1450ebf8fce488542b903 2024-11-14T00:30:07,233 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/7f3d95f337a1450ebf8fce488542b903, entries=150, sequenceid=266, filesize=30.5 K 2024-11-14T00:30:07,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/aceb86f3738b4153b57a363c33ae4cea as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/aceb86f3738b4153b57a363c33ae4cea 2024-11-14T00:30:07,249 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/aceb86f3738b4153b57a363c33ae4cea, entries=150, sequenceid=266, filesize=12.0 K 2024-11-14T00:30:07,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/503b5a199ad14dd9b572ea4f68c380bc as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/503b5a199ad14dd9b572ea4f68c380bc 2024-11-14T00:30:07,255 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/503b5a199ad14dd9b572ea4f68c380bc, entries=150, sequenceid=266, filesize=12.0 K 2024-11-14T00:30:07,260 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 4779df6820910ec85365292104f48afc in 1064ms, sequenceid=266, compaction requested=true 2024-11-14T00:30:07,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:07,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:07,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-14T00:30:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-14T00:30:07,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-14T00:30:07,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8730 sec 2024-11-14T00:30:07,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.9020 sec 2024-11-14T00:30:07,462 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:30:07,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-14T00:30:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:30:07,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114973492a07f694eef97e895ca10433c5a_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544206807/Put/seqid=0 2024-11-14T00:30:07,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742406_1582 (size=14994) 2024-11-14T00:30:07,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T00:30:07,550 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:30:07,551 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:30:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-14T00:30:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T00:30:07,559 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:07,568 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:30:07,572 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:30:07,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:30:07,585 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114973492a07f694eef97e895ca10433c5a_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114973492a07f694eef97e895ca10433c5a_4779df6820910ec85365292104f48afc 2024-11-14T00:30:07,593 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/c438a0ee93204df9b640d072dc550cce, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:07,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/c438a0ee93204df9b640d072dc550cce is 175, key is test_row_0/A:col10/1731544206807/Put/seqid=0 2024-11-14T00:30:07,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:07,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544267625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:07,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:07,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:07,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:07,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742407_1583 (size=39949) 2024-11-14T00:30:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T00:30:07,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T00:30:07,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:07,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:07,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:07,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:07,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:07,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:07,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544267745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:07,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T00:30:07,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T00:30:07,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:07,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:07,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:07,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:07,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:07,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:07,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:07,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544267959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:08,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T00:30:08,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:08,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:08,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:08,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,060 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/c438a0ee93204df9b640d072dc550cce 2024-11-14T00:30:08,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/2d1587ee997d483a8f680c7145f7c097 is 50, key is test_row_0/B:col10/1731544206807/Put/seqid=0 2024-11-14T00:30:08,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742408_1584 (size=12301) 2024-11-14T00:30:08,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T00:30:08,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T00:30:08,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:08,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:08,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:08,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544268275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:08,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T00:30:08,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:08,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:08,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:08,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/2d1587ee997d483a8f680c7145f7c097 2024-11-14T00:30:08,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T00:30:08,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:08,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:08,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:08,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/ecd442fdf70246b5a7e11e61de812ad6 is 50, key is test_row_0/C:col10/1731544206807/Put/seqid=0 2024-11-14T00:30:08,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742409_1585 (size=12301) 2024-11-14T00:30:08,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T00:30:08,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:08,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:08,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:08,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T00:30:08,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:08,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544268785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:08,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T00:30:08,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:08,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:08,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:08,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T00:30:08,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:08,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:08,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:08,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:09,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/ecd442fdf70246b5a7e11e61de812ad6 2024-11-14T00:30:09,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/c438a0ee93204df9b640d072dc550cce as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/c438a0ee93204df9b640d072dc550cce 2024-11-14T00:30:09,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/c438a0ee93204df9b640d072dc550cce, entries=200, sequenceid=288, filesize=39.0 K 2024-11-14T00:30:09,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/2d1587ee997d483a8f680c7145f7c097 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/2d1587ee997d483a8f680c7145f7c097 2024-11-14T00:30:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/2d1587ee997d483a8f680c7145f7c097, entries=150, sequenceid=288, filesize=12.0 K 2024-11-14T00:30:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/ecd442fdf70246b5a7e11e61de812ad6 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/ecd442fdf70246b5a7e11e61de812ad6 2024-11-14T00:30:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/ecd442fdf70246b5a7e11e61de812ad6, entries=150, sequenceid=288, filesize=12.0 K 2024-11-14T00:30:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 4779df6820910ec85365292104f48afc in 1621ms, sequenceid=288, compaction requested=true 2024-11-14T00:30:09,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:30:09,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:30:09,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:30:09,084 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:30:09,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:30:09,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:30:09,085 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:30:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,091 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142604 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:30:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,091 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/A is initiating minor compaction (all files) 2024-11-14T00:30:09,091 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/A in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:09,091 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:30:09,091 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/B is initiating minor compaction (all files) 2024-11-14T00:30:09,091 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/b2d7bf831f974fb0b52f536697d4144e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/c18b080dee90443da1bff516ea7b71c8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/7f3d95f337a1450ebf8fce488542b903, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/c438a0ee93204df9b640d072dc550cce] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=139.3 K 2024-11-14T00:30:09,091 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/B in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:09,091 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:09,091 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/b2d7bf831f974fb0b52f536697d4144e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/c18b080dee90443da1bff516ea7b71c8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/7f3d95f337a1450ebf8fce488542b903, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/c438a0ee93204df9b640d072dc550cce] 2024-11-14T00:30:09,092 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/a88ec517bdf24ba690b5dd1dc249b6f3, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/40cfc78ecc814e1da3efc9bc449cdc55, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/aceb86f3738b4153b57a363c33ae4cea, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/2d1587ee997d483a8f680c7145f7c097] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=48.3 K 2024-11-14T00:30:09,092 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting b2d7bf831f974fb0b52f536697d4144e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731544202598 2024-11-14T00:30:09,092 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting a88ec517bdf24ba690b5dd1dc249b6f3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731544202598 2024-11-14T00:30:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,093 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40cfc78ecc814e1da3efc9bc449cdc55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731544203957 2024-11-14T00:30:09,093 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c18b080dee90443da1bff516ea7b71c8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731544203957 2024-11-14T00:30:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,093 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting aceb86f3738b4153b57a363c33ae4cea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731544204491 2024-11-14T00:30:09,093 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f3d95f337a1450ebf8fce488542b903, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731544204491 2024-11-14T00:30:09,093 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d1587ee997d483a8f680c7145f7c097, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731544206807 2024-11-14T00:30:09,093 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c438a0ee93204df9b640d072dc550cce, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731544206804 2024-11-14T00:30:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,117 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:09,119 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:09,119 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/98a81ac9676c43be85550f04c4098ba4 is 50, key is test_row_0/B:col10/1731544206807/Put/seqid=0 2024-11-14T00:30:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,122 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114cec7d0709e7f4d5096d42d141d431efc_4779df6820910ec85365292104f48afc store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,126 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114cec7d0709e7f4d5096d42d141d431efc_4779df6820910ec85365292104f48afc, store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,126 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114cec7d0709e7f4d5096d42d141d431efc_4779df6820910ec85365292104f48afc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742410_1586 (size=12983) 2024-11-14T00:30:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T00:30:09,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:09,141 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-14T00:30:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,154 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/98a81ac9676c43be85550f04c4098ba4 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/98a81ac9676c43be85550f04c4098ba4 2024-11-14T00:30:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742411_1587 (size=4469) 2024-11-14T00:30:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,159 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#A#compaction#493 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:09,160 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/a6dc8693a845426697f71e1d3c77c820 is 175, key is test_row_0/A:col10/1731544206807/Put/seqid=0 2024-11-14T00:30:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111480e84aec5bcb4535aa4c5750ee54649e_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544207621/Put/seqid=0 2024-11-14T00:30:09,163 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4779df6820910ec85365292104f48afc/B of 4779df6820910ec85365292104f48afc into 98a81ac9676c43be85550f04c4098ba4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:09,164 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:09,164 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/B, priority=12, startTime=1731544209084; duration=0sec 2024-11-14T00:30:09,164 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:30:09,164 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:B 2024-11-14T00:30:09,164 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T00:30:09,165 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T00:30:09,165 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/C is initiating minor compaction (all files) 2024-11-14T00:30:09,165 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/C in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:09,165 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/3dda014fe80c4ebdb1122048d646f047, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/cbc835240684494da59a8f2882318987, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/503b5a199ad14dd9b572ea4f68c380bc, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/ecd442fdf70246b5a7e11e61de812ad6] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=48.3 K 2024-11-14T00:30:09,166 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dda014fe80c4ebdb1122048d646f047, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731544202598 2024-11-14T00:30:09,166 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbc835240684494da59a8f2882318987, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731544203957 2024-11-14T00:30:09,166 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 503b5a199ad14dd9b572ea4f68c380bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731544204491 2024-11-14T00:30:09,167 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecd442fdf70246b5a7e11e61de812ad6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731544206807 2024-11-14T00:30:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,187 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#C#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:09,187 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/8bbec58a71584122b8fae459fc7e99d9 is 50, key is test_row_0/C:col10/1731544206807/Put/seqid=0 2024-11-14T00:30:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742412_1588 (size=31937) 2024-11-14T00:30:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742413_1589 (size=9914) 2024-11-14T00:30:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,195 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111480e84aec5bcb4535aa4c5750ee54649e_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111480e84aec5bcb4535aa4c5750ee54649e_4779df6820910ec85365292104f48afc 2024-11-14T00:30:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/5fef02d48a394607aea0a5901cf87c30, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:09,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/5fef02d48a394607aea0a5901cf87c30 is 175, key is test_row_0/A:col10/1731544207621/Put/seqid=0 2024-11-14T00:30:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,199 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/a6dc8693a845426697f71e1d3c77c820 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/a6dc8693a845426697f71e1d3c77c820 2024-11-14T00:30:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,209 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4779df6820910ec85365292104f48afc/A of 4779df6820910ec85365292104f48afc into a6dc8693a845426697f71e1d3c77c820(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:09,209 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:09,209 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/A, priority=12, startTime=1731544209084; duration=0sec 2024-11-14T00:30:09,209 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:09,209 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:A 2024-11-14T00:30:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742414_1590 (size=12983) 2024-11-14T00:30:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742415_1591 (size=22561) 2024-11-14T00:30:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,232 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/5fef02d48a394607aea0a5901cf87c30 2024-11-14T00:30:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,263 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/8bbec58a71584122b8fae459fc7e99d9 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/8bbec58a71584122b8fae459fc7e99d9 2024-11-14T00:30:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/41bfbd13ea6b4ae1a08866c870eb5a5e is 50, key is test_row_0/B:col10/1731544207621/Put/seqid=0 2024-11-14T00:30:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,276 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4779df6820910ec85365292104f48afc/C of 4779df6820910ec85365292104f48afc into 8bbec58a71584122b8fae459fc7e99d9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:09,276 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:09,276 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/C, priority=12, startTime=1731544209084; duration=0sec 2024-11-14T00:30:09,276 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:09,276 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:C 2024-11-14T00:30:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742416_1592 (size=9857) 2024-11-14T00:30:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,303 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/41bfbd13ea6b4ae1a08866c870eb5a5e 2024-11-14T00:30:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/8ae6467854424a57a7a49b066c5bff3d is 50, key is test_row_0/C:col10/1731544207621/Put/seqid=0 2024-11-14T00:30:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742417_1593 (size=9857) 2024-11-14T00:30:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T00:30:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,729 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/8ae6467854424a57a7a49b066c5bff3d 2024-11-14T00:30:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/5fef02d48a394607aea0a5901cf87c30 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/5fef02d48a394607aea0a5901cf87c30 2024-11-14T00:30:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,745 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/5fef02d48a394607aea0a5901cf87c30, entries=100, sequenceid=302, filesize=22.0 K 2024-11-14T00:30:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/41bfbd13ea6b4ae1a08866c870eb5a5e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/41bfbd13ea6b4ae1a08866c870eb5a5e 2024-11-14T00:30:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,750 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/41bfbd13ea6b4ae1a08866c870eb5a5e, entries=100, sequenceid=302, filesize=9.6 K 2024-11-14T00:30:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/8ae6467854424a57a7a49b066c5bff3d as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/8ae6467854424a57a7a49b066c5bff3d 2024-11-14T00:30:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,757 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/8ae6467854424a57a7a49b066c5bff3d, entries=100, sequenceid=302, filesize=9.6 K 2024-11-14T00:30:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,758 INFO [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 4779df6820910ec85365292104f48afc in 617ms, sequenceid=302, compaction requested=false 2024-11-14T00:30:09,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:09,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:09,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-14T00:30:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-14T00:30:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-14T00:30:09,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1880 sec 2024-11-14T00:30:09,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.2120 sec 2024-11-14T00:30:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:09,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,040 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:30:10,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T00:30:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:30:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bf41d06b249043a5b4ec3b5cd25b526b_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544210036/Put/seqid=0 2024-11-14T00:30:10,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742418_1594 (size=12454) 2024-11-14T00:30:10,150 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:10,158 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bf41d06b249043a5b4ec3b5cd25b526b_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bf41d06b249043a5b4ec3b5cd25b526b_4779df6820910ec85365292104f48afc 2024-11-14T00:30:10,162 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/f88a54ca6f5c4191855a05e6af875058, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:10,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/f88a54ca6f5c4191855a05e6af875058 is 175, key is test_row_0/A:col10/1731544210036/Put/seqid=0 2024-11-14T00:30:10,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742419_1595 (size=31255) 2024-11-14T00:30:10,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544270324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:10,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544270435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:10,616 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/f88a54ca6f5c4191855a05e6af875058 2024-11-14T00:30:10,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/c528d6654fb849e4a60483b668f0d398 is 50, key is test_row_0/B:col10/1731544210036/Put/seqid=0 2024-11-14T00:30:10,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544270647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:10,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:10,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:10,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:10,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742420_1596 (size=12301) 2024-11-14T00:30:10,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/c528d6654fb849e4a60483b668f0d398 2024-11-14T00:30:10,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/a9d4a43025c942e5af09f3bdc5165d06 is 50, key is test_row_0/C:col10/1731544210036/Put/seqid=0 2024-11-14T00:30:10,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742421_1597 (size=12301) 2024-11-14T00:30:10,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544270956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:11,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/a9d4a43025c942e5af09f3bdc5165d06 2024-11-14T00:30:11,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/f88a54ca6f5c4191855a05e6af875058 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f88a54ca6f5c4191855a05e6af875058 2024-11-14T00:30:11,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f88a54ca6f5c4191855a05e6af875058, entries=150, sequenceid=316, filesize=30.5 K 2024-11-14T00:30:11,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/c528d6654fb849e4a60483b668f0d398 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c528d6654fb849e4a60483b668f0d398 2024-11-14T00:30:11,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c528d6654fb849e4a60483b668f0d398, entries=150, sequenceid=316, filesize=12.0 K 2024-11-14T00:30:11,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/a9d4a43025c942e5af09f3bdc5165d06 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/a9d4a43025c942e5af09f3bdc5165d06 2024-11-14T00:30:11,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/a9d4a43025c942e5af09f3bdc5165d06, entries=150, sequenceid=316, filesize=12.0 K 2024-11-14T00:30:11,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4779df6820910ec85365292104f48afc in 1132ms, sequenceid=316, compaction requested=true 2024-11-14T00:30:11,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:11,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:30:11,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:30:11,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:30:11,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T00:30:11,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:30:11,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-14T00:30:11,174 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:30:11,174 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:30:11,175 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85753 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:30:11,175 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:30:11,175 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/A is initiating minor compaction (all files) 2024-11-14T00:30:11,175 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/A in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:11,175 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/C is initiating minor compaction (all files) 2024-11-14T00:30:11,175 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/C in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:11,175 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/a6dc8693a845426697f71e1d3c77c820, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/5fef02d48a394607aea0a5901cf87c30, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f88a54ca6f5c4191855a05e6af875058] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=83.7 K 2024-11-14T00:30:11,175 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:11,175 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/8bbec58a71584122b8fae459fc7e99d9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/8ae6467854424a57a7a49b066c5bff3d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/a9d4a43025c942e5af09f3bdc5165d06] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=34.3 K 2024-11-14T00:30:11,175 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/a6dc8693a845426697f71e1d3c77c820, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/5fef02d48a394607aea0a5901cf87c30, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f88a54ca6f5c4191855a05e6af875058] 2024-11-14T00:30:11,175 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6dc8693a845426697f71e1d3c77c820, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731544206807 2024-11-14T00:30:11,176 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bbec58a71584122b8fae459fc7e99d9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731544206807 2024-11-14T00:30:11,176 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fef02d48a394607aea0a5901cf87c30, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731544207612 2024-11-14T00:30:11,176 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ae6467854424a57a7a49b066c5bff3d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731544207612 2024-11-14T00:30:11,176 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting a9d4a43025c942e5af09f3bdc5165d06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731544209984 2024-11-14T00:30:11,176 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting f88a54ca6f5c4191855a05e6af875058, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731544209984 2024-11-14T00:30:11,230 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#C#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:11,231 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/7d55f1b426014dbf8e2035b9365f9c3f is 50, key is test_row_0/C:col10/1731544210036/Put/seqid=0 2024-11-14T00:30:11,237 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:11,267 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111457d7ed6722d842e28385ac486488a3f7_4779df6820910ec85365292104f48afc store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:11,269 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111457d7ed6722d842e28385ac486488a3f7_4779df6820910ec85365292104f48afc, store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:11,269 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111457d7ed6722d842e28385ac486488a3f7_4779df6820910ec85365292104f48afc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:11,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742422_1598 (size=13085) 2024-11-14T00:30:11,320 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/7d55f1b426014dbf8e2035b9365f9c3f as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/7d55f1b426014dbf8e2035b9365f9c3f 2024-11-14T00:30:11,331 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/C of 4779df6820910ec85365292104f48afc into 7d55f1b426014dbf8e2035b9365f9c3f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:11,331 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:11,331 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/C, priority=13, startTime=1731544211173; duration=0sec 2024-11-14T00:30:11,331 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:30:11,331 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:C 2024-11-14T00:30:11,331 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:30:11,332 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:30:11,332 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/B is initiating minor compaction (all files) 2024-11-14T00:30:11,332 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/B in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:11,333 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/98a81ac9676c43be85550f04c4098ba4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/41bfbd13ea6b4ae1a08866c870eb5a5e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c528d6654fb849e4a60483b668f0d398] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=34.3 K 2024-11-14T00:30:11,333 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 98a81ac9676c43be85550f04c4098ba4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731544206807 2024-11-14T00:30:11,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742423_1599 (size=4469) 2024-11-14T00:30:11,357 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 41bfbd13ea6b4ae1a08866c870eb5a5e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731544207612 2024-11-14T00:30:11,359 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting c528d6654fb849e4a60483b668f0d398, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731544209984 2024-11-14T00:30:11,362 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#A#compaction#502 average throughput is 0.20 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:11,362 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/0bf9b4b9164a434c9e055825639b353c is 175, key is test_row_0/A:col10/1731544210036/Put/seqid=0 2024-11-14T00:30:11,382 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#B#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:11,382 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/027a87cf67014708b5420a1e6caa7782 is 50, key is test_row_0/B:col10/1731544210036/Put/seqid=0 2024-11-14T00:30:11,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742424_1600 (size=32039) 2024-11-14T00:30:11,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742425_1601 (size=13085) 2024-11-14T00:30:11,427 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/0bf9b4b9164a434c9e055825639b353c as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/0bf9b4b9164a434c9e055825639b353c 2024-11-14T00:30:11,429 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/027a87cf67014708b5420a1e6caa7782 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/027a87cf67014708b5420a1e6caa7782 2024-11-14T00:30:11,434 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/A of 4779df6820910ec85365292104f48afc into 0bf9b4b9164a434c9e055825639b353c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:11,434 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:11,434 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/A, priority=13, startTime=1731544211173; duration=0sec 2024-11-14T00:30:11,434 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:11,434 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:A 2024-11-14T00:30:11,439 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/B of 4779df6820910ec85365292104f48afc into 027a87cf67014708b5420a1e6caa7782(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:11,440 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:11,440 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/B, priority=13, startTime=1731544211173; duration=0sec 2024-11-14T00:30:11,440 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:11,440 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:B 2024-11-14T00:30:11,468 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:30:11,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-14T00:30:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:30:11,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141dfa2250901b420999c000307859a411_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544210306/Put/seqid=0 2024-11-14T00:30:11,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742426_1602 (size=14994) 2024-11-14T00:30:11,518 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:11,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:11,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544271514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:11,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:11,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:11,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:11,521 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141dfa2250901b420999c000307859a411_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141dfa2250901b420999c000307859a411_4779df6820910ec85365292104f48afc 2024-11-14T00:30:11,522 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/638865e397764f6c872ed9bb74c7d6a8, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:11,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/638865e397764f6c872ed9bb74c7d6a8 is 175, key is test_row_0/A:col10/1731544210306/Put/seqid=0 2024-11-14T00:30:11,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742427_1603 (size=39949) 2024-11-14T00:30:11,559 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/638865e397764f6c872ed9bb74c7d6a8 2024-11-14T00:30:11,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/752f471b306a444f84dc294b3c73f9bb is 50, key is test_row_0/B:col10/1731544210306/Put/seqid=0 2024-11-14T00:30:11,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:11,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544271625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:11,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:11,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742428_1604 (size=12301) 2024-11-14T00:30:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T00:30:11,717 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:30:11,718 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-14T00:30:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-14T00:30:11,719 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T00:30:11,722 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T00:30:11,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T00:30:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T00:30:11,725 INFO [Thread-2279 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2279. 2024-11-14T00:30:11,725 DEBUG [Thread-2279 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:30:11,725 DEBUG [Thread-2279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:11,725 DEBUG [Thread-2279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:11,725 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:30:11,727 INFO [Thread-2277 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2277. 2024-11-14T00:30:11,727 DEBUG [Thread-2277 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:30:11,727 DEBUG [Thread-2277 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:11,727 DEBUG [Thread-2277 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:11,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:30:11,729 INFO [Thread-2273 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2273. 2024-11-14T00:30:11,729 DEBUG [Thread-2273 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:30:11,729 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:11,729 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:11,729 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:30:11,732 INFO [Thread-2281 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2281. 2024-11-14T00:30:11,732 DEBUG [Thread-2281 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:30:11,732 DEBUG [Thread-2281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:11,732 DEBUG [Thread-2281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:11,733 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:30:11,733 INFO [Thread-2275 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2275. 2024-11-14T00:30:11,733 DEBUG [Thread-2275 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:30:11,733 DEBUG [Thread-2275 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:11,733 DEBUG [Thread-2275 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:11,733 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:30:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T00:30:11,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544271836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:11,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T00:30:11,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:11,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:11,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:11,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:11,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:11,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/752f471b306a444f84dc294b3c73f9bb 2024-11-14T00:30:12,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T00:30:12,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:12,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:12,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:12,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/1031742e77ea495fb2b14dd11f1da99e is 50, key is test_row_0/C:col10/1731544210306/Put/seqid=0 2024-11-14T00:30:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T00:30:12,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742429_1605 (size=12301) 2024-11-14T00:30:12,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T00:30:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60604 deadline: 1731544272147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 , the old value is region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T00:30:12,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., hostname=63baa11b75f3,33199,1731544045387, seqNum=5 because the exception is null or not the one we care about 2024-11-14T00:30:12,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T00:30:12,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:12,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:12,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:12,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T00:30:12,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T00:30:12,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:12,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:12,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:12,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/1031742e77ea495fb2b14dd11f1da99e 2024-11-14T00:30:12,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/638865e397764f6c872ed9bb74c7d6a8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/638865e397764f6c872ed9bb74c7d6a8 2024-11-14T00:30:12,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/638865e397764f6c872ed9bb74c7d6a8, entries=200, sequenceid=345, filesize=39.0 K 2024-11-14T00:30:12,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/752f471b306a444f84dc294b3c73f9bb as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/752f471b306a444f84dc294b3c73f9bb 2024-11-14T00:30:12,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T00:30:12,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:12,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:12,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:12,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/752f471b306a444f84dc294b3c73f9bb, entries=150, sequenceid=345, filesize=12.0 K 2024-11-14T00:30:12,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/1031742e77ea495fb2b14dd11f1da99e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/1031742e77ea495fb2b14dd11f1da99e 2024-11-14T00:30:12,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/1031742e77ea495fb2b14dd11f1da99e, entries=150, sequenceid=345, filesize=12.0 K 2024-11-14T00:30:12,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 4779df6820910ec85365292104f48afc in 1086ms, sequenceid=345, compaction requested=false 2024-11-14T00:30:12,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33199 {}] regionserver.HRegion(8855): Flush requested on 4779df6820910ec85365292104f48afc 2024-11-14T00:30:12,659 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T00:30:12,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-14T00:30:12,660 INFO [Thread-2264 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2264. 2024-11-14T00:30:12,660 DEBUG [Thread-2264 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:30:12,660 DEBUG [Thread-2264 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:12,660 DEBUG [Thread-2264 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:12,661 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:30:12,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111410db788162a945489d4a152ff8c42832_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544211501/Put/seqid=0 2024-11-14T00:30:12,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T00:30:12,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742430_1606 (size=12454) 2024-11-14T00:30:12,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:12,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:12,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:12,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T00:30:12,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:12,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:12,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:12,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T00:30:12,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T00:30:12,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:12,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:12,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:12,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:13,086 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:13,090 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111410db788162a945489d4a152ff8c42832_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111410db788162a945489d4a152ff8c42832_4779df6820910ec85365292104f48afc 2024-11-14T00:30:13,091 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/8608813359cf4d5993b9b0d00c32a94a, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:13,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/8608813359cf4d5993b9b0d00c32a94a is 175, key is test_row_0/A:col10/1731544211501/Put/seqid=0 2024-11-14T00:30:13,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742431_1607 (size=31255) 2024-11-14T00:30:13,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T00:30:13,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:13,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:13,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:13,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:13,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:13,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:13,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T00:30:13,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:13,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:13,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:13,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:13,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:13,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T00:30:13,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:13,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. as already flushing 2024-11-14T00:30:13,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:13,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:13,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T00:30:13,496 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/8608813359cf4d5993b9b0d00c32a94a 2024-11-14T00:30:13,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/68f2ff37a57040b99370a62121a50f59 is 50, key is test_row_0/B:col10/1731544211501/Put/seqid=0 2024-11-14T00:30:13,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742432_1608 (size=12301) 2024-11-14T00:30:13,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/68f2ff37a57040b99370a62121a50f59 2024-11-14T00:30:13,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/adc52a4f80cf4e909775857f2a88170e is 50, key is test_row_0/C:col10/1731544211501/Put/seqid=0 2024-11-14T00:30:13,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742433_1609 (size=12301) 2024-11-14T00:30:13,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/adc52a4f80cf4e909775857f2a88170e 2024-11-14T00:30:13,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/8608813359cf4d5993b9b0d00c32a94a as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8608813359cf4d5993b9b0d00c32a94a 2024-11-14T00:30:13,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8608813359cf4d5993b9b0d00c32a94a, entries=150, sequenceid=356, filesize=30.5 K 2024-11-14T00:30:13,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/68f2ff37a57040b99370a62121a50f59 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/68f2ff37a57040b99370a62121a50f59 2024-11-14T00:30:13,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/68f2ff37a57040b99370a62121a50f59, entries=150, sequenceid=356, filesize=12.0 K 2024-11-14T00:30:13,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/adc52a4f80cf4e909775857f2a88170e as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/adc52a4f80cf4e909775857f2a88170e 2024-11-14T00:30:13,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/adc52a4f80cf4e909775857f2a88170e, entries=150, sequenceid=356, filesize=12.0 K 2024-11-14T00:30:13,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 4779df6820910ec85365292104f48afc in 919ms, sequenceid=356, compaction requested=true 2024-11-14T00:30:13,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:13,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T00:30:13,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:13,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T00:30:13,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:13,579 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:30:13,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4779df6820910ec85365292104f48afc:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T00:30:13,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:30:13,579 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:30:13,580 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:30:13,580 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:30:13,580 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/B is initiating minor compaction (all files) 2024-11-14T00:30:13,580 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/A is initiating minor compaction (all files) 2024-11-14T00:30:13,580 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/A in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:13,580 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/B in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:13,580 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/027a87cf67014708b5420a1e6caa7782, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/752f471b306a444f84dc294b3c73f9bb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/68f2ff37a57040b99370a62121a50f59] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=36.8 K 2024-11-14T00:30:13,580 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/0bf9b4b9164a434c9e055825639b353c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/638865e397764f6c872ed9bb74c7d6a8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8608813359cf4d5993b9b0d00c32a94a] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=100.8 K 2024-11-14T00:30:13,580 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:13,580 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. files: [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/0bf9b4b9164a434c9e055825639b353c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/638865e397764f6c872ed9bb74c7d6a8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8608813359cf4d5993b9b0d00c32a94a] 2024-11-14T00:30:13,581 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bf9b4b9164a434c9e055825639b353c, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731544209984 2024-11-14T00:30:13,581 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 027a87cf67014708b5420a1e6caa7782, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731544209984 2024-11-14T00:30:13,581 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 638865e397764f6c872ed9bb74c7d6a8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731544210299 2024-11-14T00:30:13,581 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 752f471b306a444f84dc294b3c73f9bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731544210299 2024-11-14T00:30:13,581 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8608813359cf4d5993b9b0d00c32a94a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731544211501 2024-11-14T00:30:13,581 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 68f2ff37a57040b99370a62121a50f59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731544211501 2024-11-14T00:30:13,592 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#B#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:13,593 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/d2affb4612454546a66f745dcf36ca45 is 50, key is test_row_0/B:col10/1731544211501/Put/seqid=0 2024-11-14T00:30:13,594 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:13,606 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411143712ba8dabbb4e729da3dec1299bcf19_4779df6820910ec85365292104f48afc store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:13,611 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411143712ba8dabbb4e729da3dec1299bcf19_4779df6820910ec85365292104f48afc, store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:13,611 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143712ba8dabbb4e729da3dec1299bcf19_4779df6820910ec85365292104f48afc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:13,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33199 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T00:30:13,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:13,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:13,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:13,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/63baa11b75f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-14T00:30:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-14T00:30:13,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742434_1610 (size=13187) 2024-11-14T00:30:13,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-14T00:30:13,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9050 sec 2024-11-14T00:30:13,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.9260 sec 2024-11-14T00:30:13,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742435_1611 (size=4469) 2024-11-14T00:30:13,661 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#A#compaction#511 average throughput is 0.37 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:13,661 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/a001f40b54cb4da2beb99f9ba45b13c2 is 175, key is test_row_0/A:col10/1731544211501/Put/seqid=0 2024-11-14T00:30:13,672 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/d2affb4612454546a66f745dcf36ca45 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/d2affb4612454546a66f745dcf36ca45 2024-11-14T00:30:13,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742436_1612 (size=32141) 2024-11-14T00:30:13,700 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/B of 4779df6820910ec85365292104f48afc into d2affb4612454546a66f745dcf36ca45(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:13,701 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:13,701 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/B, priority=13, startTime=1731544213579; duration=0sec 2024-11-14T00:30:13,701 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T00:30:13,701 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:B 2024-11-14T00:30:13,701 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T00:30:13,703 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T00:30:13,703 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1541): 4779df6820910ec85365292104f48afc/C is initiating minor compaction (all files) 2024-11-14T00:30:13,703 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4779df6820910ec85365292104f48afc/C in TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:13,703 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/7d55f1b426014dbf8e2035b9365f9c3f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/1031742e77ea495fb2b14dd11f1da99e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/adc52a4f80cf4e909775857f2a88170e] into tmpdir=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp, totalSize=36.8 K 2024-11-14T00:30:13,703 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d55f1b426014dbf8e2035b9365f9c3f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731544209984 2024-11-14T00:30:13,704 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting 1031742e77ea495fb2b14dd11f1da99e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731544210299 2024-11-14T00:30:13,704 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] compactions.Compactor(225): Compacting adc52a4f80cf4e909775857f2a88170e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731544211501 2024-11-14T00:30:13,718 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4779df6820910ec85365292104f48afc#C#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T00:30:13,719 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/c76127ddaa0c42ae85327cd75ff6a2d6 is 50, key is test_row_0/C:col10/1731544211501/Put/seqid=0 2024-11-14T00:30:13,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742437_1613 (size=13187) 2024-11-14T00:30:13,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T00:30:13,876 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:30:14,089 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/a001f40b54cb4da2beb99f9ba45b13c2 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/a001f40b54cb4da2beb99f9ba45b13c2 2024-11-14T00:30:14,093 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/A of 4779df6820910ec85365292104f48afc into a001f40b54cb4da2beb99f9ba45b13c2(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:14,093 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:14,093 INFO [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/A, priority=13, startTime=1731544213578; duration=0sec 2024-11-14T00:30:14,093 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:14,093 DEBUG [RS:0;63baa11b75f3:33199-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:A 2024-11-14T00:30:14,138 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/c76127ddaa0c42ae85327cd75ff6a2d6 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/c76127ddaa0c42ae85327cd75ff6a2d6 2024-11-14T00:30:14,142 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4779df6820910ec85365292104f48afc/C of 4779df6820910ec85365292104f48afc into c76127ddaa0c42ae85327cd75ff6a2d6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T00:30:14,143 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4779df6820910ec85365292104f48afc: 2024-11-14T00:30:14,143 INFO [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., storeName=4779df6820910ec85365292104f48afc/C, priority=13, startTime=1731544213579; duration=0sec 2024-11-14T00:30:14,143 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T00:30:14,143 DEBUG [RS:0;63baa11b75f3:33199-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4779df6820910ec85365292104f48afc:C 2024-11-14T00:30:14,187 INFO [Thread-2266 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2266. 2024-11-14T00:30:14,187 DEBUG [Thread-2266 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:30:14,187 DEBUG [Thread-2266 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:14,187 DEBUG [Thread-2266 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:14,187 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:30:14,207 INFO [Thread-2270 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2270. 2024-11-14T00:30:14,207 DEBUG [Thread-2270 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:30:14,207 DEBUG [Thread-2270 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:14,207 DEBUG [Thread-2270 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:14,207 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:30:14,237 INFO [Thread-2262 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2262. 2024-11-14T00:30:14,237 DEBUG [Thread-2262 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:30:14,237 DEBUG [Thread-2262 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:14,237 DEBUG [Thread-2262 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:14,237 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:30:14,276 INFO [Thread-2268 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2268. 2024-11-14T00:30:14,276 DEBUG [Thread-2268 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T00:30:14,276 DEBUG [Thread-2268 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:14,276 DEBUG [Thread-2268 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:14,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:30:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T00:30:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-14T00:30:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 207 2024-11-14T00:30:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-14T00:30:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-14T00:30:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-14T00:30:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T00:30:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2317 2024-11-14T00:30:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2285 2024-11-14T00:30:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2307 2024-11-14T00:30:14,276 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2303 2024-11-14T00:30:14,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2273 2024-11-14T00:30:14,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T00:30:14,277 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T00:30:14,277 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T00:30:14,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:14,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:14,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:30:14,277 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-14T00:30:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T00:30:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T00:30:14,280 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544214280"}]},"ts":"1731544214280"} 2024-11-14T00:30:14,282 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T00:30:14,282 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T00:30:14,282 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T00:30:14,283 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4779df6820910ec85365292104f48afc, UNASSIGN}] 2024-11-14T00:30:14,284 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4779df6820910ec85365292104f48afc, UNASSIGN 2024-11-14T00:30:14,285 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=4779df6820910ec85365292104f48afc, regionState=CLOSING, regionLocation=63baa11b75f3,33199,1731544045387 2024-11-14T00:30:14,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4779df6820910ec85365292104f48afc, UNASSIGN because future has completed 2024-11-14T00:30:14,286 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T00:30:14,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387}] 2024-11-14T00:30:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T00:30:14,438 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(122): Close 4779df6820910ec85365292104f48afc 2024-11-14T00:30:14,438 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T00:30:14,438 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1722): Closing 4779df6820910ec85365292104f48afc, disabling compactions & flushes 2024-11-14T00:30:14,438 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:14,438 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:14,438 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. after waiting 0 ms 2024-11-14T00:30:14,438 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:14,439 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(2902): Flushing 4779df6820910ec85365292104f48afc 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-14T00:30:14,445 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d09c44b2bf3d428998b1f9f47be8dea7_4779df6820910ec85365292104f48afc is 50, key is test_row_0/A:col10/1731544214235/Put/seqid=0 2024-11-14T00:30:14,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742438_1614 (size=12454) 2024-11-14T00:30:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T00:30:14,860 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T00:30:14,864 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d09c44b2bf3d428998b1f9f47be8dea7_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d09c44b2bf3d428998b1f9f47be8dea7_4779df6820910ec85365292104f48afc 2024-11-14T00:30:14,868 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/8bee7d86bc994a749af4f4cdb6879ff3, store: [table=TestAcidGuarantees family=A region=4779df6820910ec85365292104f48afc] 2024-11-14T00:30:14,869 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/8bee7d86bc994a749af4f4cdb6879ff3 is 175, key is test_row_0/A:col10/1731544214235/Put/seqid=0 2024-11-14T00:30:14,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742439_1615 (size=31255) 2024-11-14T00:30:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T00:30:15,275 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/8bee7d86bc994a749af4f4cdb6879ff3 2024-11-14T00:30:15,281 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/bf12c0276dc34911abe278b22ae8eee8 is 50, key is test_row_0/B:col10/1731544214235/Put/seqid=0 2024-11-14T00:30:15,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742440_1616 (size=12301) 2024-11-14T00:30:15,294 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/bf12c0276dc34911abe278b22ae8eee8 2024-11-14T00:30:15,303 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/16905d0ffb1d4d10a3a7d35e206923bf is 50, key is test_row_0/C:col10/1731544214235/Put/seqid=0 2024-11-14T00:30:15,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742441_1617 (size=12301) 2024-11-14T00:30:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T00:30:15,712 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/16905d0ffb1d4d10a3a7d35e206923bf 2024-11-14T00:30:15,716 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/A/8bee7d86bc994a749af4f4cdb6879ff3 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8bee7d86bc994a749af4f4cdb6879ff3 2024-11-14T00:30:15,719 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8bee7d86bc994a749af4f4cdb6879ff3, entries=150, sequenceid=367, filesize=30.5 K 2024-11-14T00:30:15,720 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/B/bf12c0276dc34911abe278b22ae8eee8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/bf12c0276dc34911abe278b22ae8eee8 2024-11-14T00:30:15,724 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/bf12c0276dc34911abe278b22ae8eee8, entries=150, sequenceid=367, filesize=12.0 K 2024-11-14T00:30:15,725 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/.tmp/C/16905d0ffb1d4d10a3a7d35e206923bf as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/16905d0ffb1d4d10a3a7d35e206923bf 2024-11-14T00:30:15,728 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/16905d0ffb1d4d10a3a7d35e206923bf, entries=150, sequenceid=367, filesize=12.0 K 2024-11-14T00:30:15,729 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 4779df6820910ec85365292104f48afc in 1291ms, sequenceid=367, compaction requested=false 2024-11-14T00:30:15,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8c258ad32d3a4778a2cfbf0b587dcd9e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/42b1f8a8981941bc992a9680884c0b51, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/02b86f8fb41c4c28aa60a5dde8c3ab2d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/fac400b433f74d89915ac66ce78ffb95, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/7b5ecb4ff2e346188f6aa0679565e473, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f21d209fa6124ec6b28134ff1c28b17f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/27c3514067144c21adee605f741bdb58, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f56899df841d4b20927371bfa52cb30b, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/734ce039ebc448c6a432b44099dc1afc, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f90d72489c9b4a8cbd568ea366822623, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/a0619950247d4b41b6de3db031b86536, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/64191ad002934f1ab8a212b3d485be2f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/84df7f6252df4037b8c8d08b23ad3c07, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/0bcf27a4635a4bb59f18aeec64da9652, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/3a61cc99ee2b42c395ec94353dca864a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/b2d7bf831f974fb0b52f536697d4144e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/9ffcf9fd87fe45d6ae3dbf84e42dddd8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/c18b080dee90443da1bff516ea7b71c8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/7f3d95f337a1450ebf8fce488542b903, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/c438a0ee93204df9b640d072dc550cce, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/a6dc8693a845426697f71e1d3c77c820, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/5fef02d48a394607aea0a5901cf87c30, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/0bf9b4b9164a434c9e055825639b353c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f88a54ca6f5c4191855a05e6af875058, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/638865e397764f6c872ed9bb74c7d6a8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8608813359cf4d5993b9b0d00c32a94a] to archive 2024-11-14T00:30:15,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:30:15,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8c258ad32d3a4778a2cfbf0b587dcd9e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8c258ad32d3a4778a2cfbf0b587dcd9e 2024-11-14T00:30:15,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/42b1f8a8981941bc992a9680884c0b51 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/42b1f8a8981941bc992a9680884c0b51 2024-11-14T00:30:15,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/02b86f8fb41c4c28aa60a5dde8c3ab2d to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/02b86f8fb41c4c28aa60a5dde8c3ab2d 2024-11-14T00:30:15,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/fac400b433f74d89915ac66ce78ffb95 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/fac400b433f74d89915ac66ce78ffb95 2024-11-14T00:30:15,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/7b5ecb4ff2e346188f6aa0679565e473 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/7b5ecb4ff2e346188f6aa0679565e473 2024-11-14T00:30:15,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f21d209fa6124ec6b28134ff1c28b17f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f21d209fa6124ec6b28134ff1c28b17f 2024-11-14T00:30:15,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/27c3514067144c21adee605f741bdb58 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/27c3514067144c21adee605f741bdb58 2024-11-14T00:30:15,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f56899df841d4b20927371bfa52cb30b to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f56899df841d4b20927371bfa52cb30b 2024-11-14T00:30:15,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/734ce039ebc448c6a432b44099dc1afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/734ce039ebc448c6a432b44099dc1afc 2024-11-14T00:30:15,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f90d72489c9b4a8cbd568ea366822623 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f90d72489c9b4a8cbd568ea366822623 2024-11-14T00:30:15,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/a0619950247d4b41b6de3db031b86536 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/a0619950247d4b41b6de3db031b86536 2024-11-14T00:30:15,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/64191ad002934f1ab8a212b3d485be2f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/64191ad002934f1ab8a212b3d485be2f 2024-11-14T00:30:15,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/84df7f6252df4037b8c8d08b23ad3c07 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/84df7f6252df4037b8c8d08b23ad3c07 2024-11-14T00:30:15,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/0bcf27a4635a4bb59f18aeec64da9652 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/0bcf27a4635a4bb59f18aeec64da9652 2024-11-14T00:30:15,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/3a61cc99ee2b42c395ec94353dca864a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/3a61cc99ee2b42c395ec94353dca864a 2024-11-14T00:30:15,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/b2d7bf831f974fb0b52f536697d4144e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/b2d7bf831f974fb0b52f536697d4144e 2024-11-14T00:30:15,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/9ffcf9fd87fe45d6ae3dbf84e42dddd8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/9ffcf9fd87fe45d6ae3dbf84e42dddd8 2024-11-14T00:30:15,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/c18b080dee90443da1bff516ea7b71c8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/c18b080dee90443da1bff516ea7b71c8 2024-11-14T00:30:15,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/7f3d95f337a1450ebf8fce488542b903 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/7f3d95f337a1450ebf8fce488542b903 2024-11-14T00:30:15,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/c438a0ee93204df9b640d072dc550cce to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/c438a0ee93204df9b640d072dc550cce 2024-11-14T00:30:15,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/a6dc8693a845426697f71e1d3c77c820 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/a6dc8693a845426697f71e1d3c77c820 2024-11-14T00:30:15,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/5fef02d48a394607aea0a5901cf87c30 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/5fef02d48a394607aea0a5901cf87c30 2024-11-14T00:30:15,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/0bf9b4b9164a434c9e055825639b353c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/0bf9b4b9164a434c9e055825639b353c 2024-11-14T00:30:15,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f88a54ca6f5c4191855a05e6af875058 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/f88a54ca6f5c4191855a05e6af875058 2024-11-14T00:30:15,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/638865e397764f6c872ed9bb74c7d6a8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/638865e397764f6c872ed9bb74c7d6a8 2024-11-14T00:30:15,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8608813359cf4d5993b9b0d00c32a94a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8608813359cf4d5993b9b0d00c32a94a 2024-11-14T00:30:15,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/fc9ab799c11145a995265e75b2737323, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/9137700e0c4a41a2a29f3a5670b21925, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/d6cb1e7aef4b4ddabf5dd7e75f4b6cae, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/2e6f19a014a34afc944b2a8adb1a11dd, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/b0f9754c9a22463ca49f39927c101581, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/1b2110f3ceed469cae4396a2effcc90a, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c08f18db2e8b43cca5dc73dd1ad1617f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c01ab9231b364f8bb9b46b8dd7c152d7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c79b577e3c514130aa5bbca6f03e99bd, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/ef3b574a71614ad9a8edca261432fa49, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/f0e042809dc640beace0ccf2be6665fd, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/6870907f9d3e4161968250e78f257491, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/9421a05a2384418baa178f176d5efba2, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/adc624618b1c42c8b4c52cc4dcc9575c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/1ea81481eb78409d8a79d7bb6fba4658, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/a88ec517bdf24ba690b5dd1dc249b6f3, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/484150d8aad443e48e6cde9d5559e9f7, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/40cfc78ecc814e1da3efc9bc449cdc55, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/aceb86f3738b4153b57a363c33ae4cea, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/98a81ac9676c43be85550f04c4098ba4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/2d1587ee997d483a8f680c7145f7c097, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/41bfbd13ea6b4ae1a08866c870eb5a5e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/027a87cf67014708b5420a1e6caa7782, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c528d6654fb849e4a60483b668f0d398, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/752f471b306a444f84dc294b3c73f9bb, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/68f2ff37a57040b99370a62121a50f59] to archive 2024-11-14T00:30:15,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:30:15,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/fc9ab799c11145a995265e75b2737323 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/fc9ab799c11145a995265e75b2737323 2024-11-14T00:30:15,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/9137700e0c4a41a2a29f3a5670b21925 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/9137700e0c4a41a2a29f3a5670b21925 2024-11-14T00:30:15,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/d6cb1e7aef4b4ddabf5dd7e75f4b6cae to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/d6cb1e7aef4b4ddabf5dd7e75f4b6cae 2024-11-14T00:30:15,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/2e6f19a014a34afc944b2a8adb1a11dd to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/2e6f19a014a34afc944b2a8adb1a11dd 2024-11-14T00:30:15,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/b0f9754c9a22463ca49f39927c101581 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/b0f9754c9a22463ca49f39927c101581 2024-11-14T00:30:15,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/1b2110f3ceed469cae4396a2effcc90a to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/1b2110f3ceed469cae4396a2effcc90a 2024-11-14T00:30:15,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c08f18db2e8b43cca5dc73dd1ad1617f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c08f18db2e8b43cca5dc73dd1ad1617f 2024-11-14T00:30:15,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c01ab9231b364f8bb9b46b8dd7c152d7 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c01ab9231b364f8bb9b46b8dd7c152d7 2024-11-14T00:30:15,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c79b577e3c514130aa5bbca6f03e99bd to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c79b577e3c514130aa5bbca6f03e99bd 2024-11-14T00:30:15,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/ef3b574a71614ad9a8edca261432fa49 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/ef3b574a71614ad9a8edca261432fa49 2024-11-14T00:30:15,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/f0e042809dc640beace0ccf2be6665fd to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/f0e042809dc640beace0ccf2be6665fd 2024-11-14T00:30:15,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/6870907f9d3e4161968250e78f257491 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/6870907f9d3e4161968250e78f257491 2024-11-14T00:30:15,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/9421a05a2384418baa178f176d5efba2 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/9421a05a2384418baa178f176d5efba2 2024-11-14T00:30:15,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/adc624618b1c42c8b4c52cc4dcc9575c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/adc624618b1c42c8b4c52cc4dcc9575c 2024-11-14T00:30:15,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/1ea81481eb78409d8a79d7bb6fba4658 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/1ea81481eb78409d8a79d7bb6fba4658 2024-11-14T00:30:15,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/a88ec517bdf24ba690b5dd1dc249b6f3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/a88ec517bdf24ba690b5dd1dc249b6f3 2024-11-14T00:30:15,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/484150d8aad443e48e6cde9d5559e9f7 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/484150d8aad443e48e6cde9d5559e9f7 2024-11-14T00:30:15,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/40cfc78ecc814e1da3efc9bc449cdc55 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/40cfc78ecc814e1da3efc9bc449cdc55 2024-11-14T00:30:15,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/aceb86f3738b4153b57a363c33ae4cea to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/aceb86f3738b4153b57a363c33ae4cea 2024-11-14T00:30:15,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/98a81ac9676c43be85550f04c4098ba4 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/98a81ac9676c43be85550f04c4098ba4 2024-11-14T00:30:15,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/2d1587ee997d483a8f680c7145f7c097 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/2d1587ee997d483a8f680c7145f7c097 2024-11-14T00:30:15,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/41bfbd13ea6b4ae1a08866c870eb5a5e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/41bfbd13ea6b4ae1a08866c870eb5a5e 2024-11-14T00:30:15,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/027a87cf67014708b5420a1e6caa7782 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/027a87cf67014708b5420a1e6caa7782 2024-11-14T00:30:15,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c528d6654fb849e4a60483b668f0d398 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/c528d6654fb849e4a60483b668f0d398 2024-11-14T00:30:15,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/752f471b306a444f84dc294b3c73f9bb to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/752f471b306a444f84dc294b3c73f9bb 2024-11-14T00:30:15,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/68f2ff37a57040b99370a62121a50f59 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/68f2ff37a57040b99370a62121a50f59 2024-11-14T00:30:15,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/5bab2d396e5b4597bc2fb3d5fe415994, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/be215ea3ce9c4d15b15481ffd9ac4211, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/a96954346e9e49e691ba0b0d0dcb658d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/8d388174faa943bd981d4f75e46d21de, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/351938c3193b4cb482d10efcea079c18, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/fa07c3a984794f5094155185075df340, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/14f2d167de0747c89350627172f61587, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/f24c17edf7c8459d95cadd21f53145c6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/5bb133621e6f484dbdc0d72aed79a0d8, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/157972e557674bf8a59ef50cc2f0172c, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/43bcb3e1ea82480b864430c4e3ba53ab, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/9349c4564be14c7c83662218842b3297, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/0ab73e832ef44e6db24b5f6249f8d3e4, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/d03a720a34e644babb9c93eda8e2690e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/9f0fca09f0314beb88e85225face8c45, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/3dda014fe80c4ebdb1122048d646f047, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/db6518564c074c578fb134e2e36873db, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/cbc835240684494da59a8f2882318987, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/503b5a199ad14dd9b572ea4f68c380bc, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/8bbec58a71584122b8fae459fc7e99d9, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/ecd442fdf70246b5a7e11e61de812ad6, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/8ae6467854424a57a7a49b066c5bff3d, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/7d55f1b426014dbf8e2035b9365f9c3f, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/a9d4a43025c942e5af09f3bdc5165d06, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/1031742e77ea495fb2b14dd11f1da99e, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/adc52a4f80cf4e909775857f2a88170e] to archive 2024-11-14T00:30:15,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T00:30:15,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/5bab2d396e5b4597bc2fb3d5fe415994 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/5bab2d396e5b4597bc2fb3d5fe415994 2024-11-14T00:30:15,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/be215ea3ce9c4d15b15481ffd9ac4211 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/be215ea3ce9c4d15b15481ffd9ac4211 2024-11-14T00:30:15,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/a96954346e9e49e691ba0b0d0dcb658d to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/a96954346e9e49e691ba0b0d0dcb658d 2024-11-14T00:30:15,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/8d388174faa943bd981d4f75e46d21de to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/8d388174faa943bd981d4f75e46d21de 2024-11-14T00:30:15,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/351938c3193b4cb482d10efcea079c18 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/351938c3193b4cb482d10efcea079c18 2024-11-14T00:30:15,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/fa07c3a984794f5094155185075df340 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/fa07c3a984794f5094155185075df340 2024-11-14T00:30:15,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/14f2d167de0747c89350627172f61587 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/14f2d167de0747c89350627172f61587 2024-11-14T00:30:15,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/f24c17edf7c8459d95cadd21f53145c6 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/f24c17edf7c8459d95cadd21f53145c6 2024-11-14T00:30:15,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/5bb133621e6f484dbdc0d72aed79a0d8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/5bb133621e6f484dbdc0d72aed79a0d8 2024-11-14T00:30:15,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/157972e557674bf8a59ef50cc2f0172c to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/157972e557674bf8a59ef50cc2f0172c 2024-11-14T00:30:15,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/43bcb3e1ea82480b864430c4e3ba53ab to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/43bcb3e1ea82480b864430c4e3ba53ab 2024-11-14T00:30:15,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/9349c4564be14c7c83662218842b3297 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/9349c4564be14c7c83662218842b3297 2024-11-14T00:30:15,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/0ab73e832ef44e6db24b5f6249f8d3e4 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/0ab73e832ef44e6db24b5f6249f8d3e4 2024-11-14T00:30:15,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/d03a720a34e644babb9c93eda8e2690e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/d03a720a34e644babb9c93eda8e2690e 2024-11-14T00:30:15,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/9f0fca09f0314beb88e85225face8c45 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/9f0fca09f0314beb88e85225face8c45 2024-11-14T00:30:15,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/3dda014fe80c4ebdb1122048d646f047 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/3dda014fe80c4ebdb1122048d646f047 2024-11-14T00:30:15,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/db6518564c074c578fb134e2e36873db to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/db6518564c074c578fb134e2e36873db 2024-11-14T00:30:15,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/cbc835240684494da59a8f2882318987 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/cbc835240684494da59a8f2882318987 2024-11-14T00:30:15,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/503b5a199ad14dd9b572ea4f68c380bc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/503b5a199ad14dd9b572ea4f68c380bc 2024-11-14T00:30:15,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/8bbec58a71584122b8fae459fc7e99d9 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/8bbec58a71584122b8fae459fc7e99d9 2024-11-14T00:30:15,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/ecd442fdf70246b5a7e11e61de812ad6 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/ecd442fdf70246b5a7e11e61de812ad6 2024-11-14T00:30:15,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/8ae6467854424a57a7a49b066c5bff3d to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/8ae6467854424a57a7a49b066c5bff3d 2024-11-14T00:30:15,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/7d55f1b426014dbf8e2035b9365f9c3f to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/7d55f1b426014dbf8e2035b9365f9c3f 2024-11-14T00:30:15,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/a9d4a43025c942e5af09f3bdc5165d06 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/a9d4a43025c942e5af09f3bdc5165d06 2024-11-14T00:30:15,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/1031742e77ea495fb2b14dd11f1da99e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/1031742e77ea495fb2b14dd11f1da99e 2024-11-14T00:30:15,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/adc52a4f80cf4e909775857f2a88170e to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/adc52a4f80cf4e909775857f2a88170e 2024-11-14T00:30:15,830 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/recovered.edits/370.seqid, newMaxSeqId=370, maxSeqId=4 2024-11-14T00:30:15,831 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. 2024-11-14T00:30:15,831 DEBUG [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1676): Region close journal for 4779df6820910ec85365292104f48afc: Waiting for close lock at 1731544214438Running coprocessor pre-close hooks at 1731544214438Disabling compacts and flushes for region at 1731544214438Disabling writes for close at 1731544214438Obtaining lock to block concurrent updates at 1731544214439 (+1 ms)Preparing flush snapshotting stores in 4779df6820910ec85365292104f48afc at 1731544214439Finished memstore snapshotting TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731544214439Flushing stores of TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc. at 1731544214439Flushing 4779df6820910ec85365292104f48afc/A: creating writer at 1731544214439Flushing mob file 4779df6820910ec85365292104f48afc/A: appending metadata at 1731544214445 (+6 ms)Flushing mob file 4779df6820910ec85365292104f48afc/A: closing flushed file at 1731544214445Flushing 4779df6820910ec85365292104f48afc/A: appending metadata at 1731544214868 (+423 ms)Flushing 4779df6820910ec85365292104f48afc/A: closing flushed file at 1731544214869 (+1 ms)Flushing 4779df6820910ec85365292104f48afc/B: creating writer at 1731544215278 (+409 ms)Flushing 4779df6820910ec85365292104f48afc/B: appending metadata at 1731544215280 (+2 ms)Flushing 4779df6820910ec85365292104f48afc/B: closing flushed file at 1731544215281 (+1 ms)Flushing 4779df6820910ec85365292104f48afc/C: creating writer at 1731544215300 (+19 ms)Flushing 4779df6820910ec85365292104f48afc/C: appending metadata at 1731544215302 (+2 ms)Flushing 4779df6820910ec85365292104f48afc/C: closing flushed file at 1731544215302Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@471c32f9: reopening flushed file at 1731544215715 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6552a99f: reopening flushed file at 1731544215720 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31420146: reopening flushed file at 1731544215724 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 4779df6820910ec85365292104f48afc in 1291ms, sequenceid=367, compaction requested=false at 1731544215729 (+5 ms)Writing region close event to WAL at 1731544215827 (+98 ms)Running coprocessor post-close hooks at 1731544215830 (+3 ms)Closed at 1731544215831 (+1 ms) 2024-11-14T00:30:15,833 INFO [RS_CLOSE_REGION-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(157): Closed 4779df6820910ec85365292104f48afc 2024-11-14T00:30:15,833 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=4779df6820910ec85365292104f48afc, regionState=CLOSED 2024-11-14T00:30:15,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 because future has completed 2024-11-14T00:30:15,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-14T00:30:15,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseRegionProcedure 4779df6820910ec85365292104f48afc, server=63baa11b75f3,33199,1731544045387 in 1.5490 sec 2024-11-14T00:30:15,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-14T00:30:15,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4779df6820910ec85365292104f48afc, UNASSIGN in 1.5540 sec 2024-11-14T00:30:15,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-14T00:30:15,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5580 sec 2024-11-14T00:30:15,844 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731544215843"}]},"ts":"1731544215843"} 2024-11-14T00:30:15,845 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T00:30:15,845 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T00:30:15,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5680 sec 2024-11-14T00:30:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T00:30:16,426 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:30:16,426 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-14T00:30:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:30:16,428 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:30:16,429 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=162, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:30:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T00:30:16,432 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,434 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C, FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/recovered.edits] 2024-11-14T00:30:16,437 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8bee7d86bc994a749af4f4cdb6879ff3 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/8bee7d86bc994a749af4f4cdb6879ff3 2024-11-14T00:30:16,438 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/a001f40b54cb4da2beb99f9ba45b13c2 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/A/a001f40b54cb4da2beb99f9ba45b13c2 2024-11-14T00:30:16,440 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/bf12c0276dc34911abe278b22ae8eee8 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/bf12c0276dc34911abe278b22ae8eee8 2024-11-14T00:30:16,441 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/d2affb4612454546a66f745dcf36ca45 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/B/d2affb4612454546a66f745dcf36ca45 2024-11-14T00:30:16,445 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/16905d0ffb1d4d10a3a7d35e206923bf to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/16905d0ffb1d4d10a3a7d35e206923bf 2024-11-14T00:30:16,448 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/c76127ddaa0c42ae85327cd75ff6a2d6 to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/C/c76127ddaa0c42ae85327cd75ff6a2d6 2024-11-14T00:30:16,452 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/recovered.edits/370.seqid to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc/recovered.edits/370.seqid 2024-11-14T00:30:16,452 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/default/TestAcidGuarantees/4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,453 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T00:30:16,453 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T00:30:16,454 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T00:30:16,456 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111410db788162a945489d4a152ff8c42832_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111410db788162a945489d4a152ff8c42832_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,458 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141768129383f148e48d213df20f82b8a1_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141768129383f148e48d213df20f82b8a1_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,459 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111418feeda1f153424ebf0dac286ca0e492_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111418feeda1f153424ebf0dac286ca0e492_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,460 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141dfa2250901b420999c000307859a411_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141dfa2250901b420999c000307859a411_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,462 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142e67f817a3324eb1abe7b78c00627c3e_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142e67f817a3324eb1abe7b78c00627c3e_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,463 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111446c7738b4b174f4088e10a5d2bafe844_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111446c7738b4b174f4088e10a5d2bafe844_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,464 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144d0392c4dd924b33880892cc9d56bdbc_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144d0392c4dd924b33880892cc9d56bdbc_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,466 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147c5427c0fa324a06883ef8b231bb775f_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147c5427c0fa324a06883ef8b231bb775f_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,467 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111480e84aec5bcb4535aa4c5750ee54649e_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111480e84aec5bcb4535aa4c5750ee54649e_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,468 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111495268acfed714a7c895c7d7104697823_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111495268acfed714a7c895c7d7104697823_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,469 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114973492a07f694eef97e895ca10433c5a_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114973492a07f694eef97e895ca10433c5a_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,470 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149b7d90c91a794d658da2eacc921742d8_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149b7d90c91a794d658da2eacc921742d8_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,471 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b8b7a6c413424d05a98f2a7e7aacdb75_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b8b7a6c413424d05a98f2a7e7aacdb75_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,472 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bab61fc7b4494f05948ba1b25b82e823_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bab61fc7b4494f05948ba1b25b82e823_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,476 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bda3f6eb2918427e8ec78ba2277a0ab5_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bda3f6eb2918427e8ec78ba2277a0ab5_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,481 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bf41d06b249043a5b4ec3b5cd25b526b_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bf41d06b249043a5b4ec3b5cd25b526b_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,484 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d09c44b2bf3d428998b1f9f47be8dea7_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d09c44b2bf3d428998b1f9f47be8dea7_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,487 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e606835734f74a21b69dbf4ab2c25f13_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e606835734f74a21b69dbf4ab2c25f13_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,489 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb3bb008c2b34f418ce0f74417a64b1c_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb3bb008c2b34f418ce0f74417a64b1c_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,490 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fe6bc01e1826493ab702b44837154381_4779df6820910ec85365292104f48afc to hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114fe6bc01e1826493ab702b44837154381_4779df6820910ec85365292104f48afc 2024-11-14T00:30:16,490 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T00:30:16,492 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=162, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:30:16,495 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T00:30:16,498 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T00:30:16,499 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=162, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:30:16,499 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T00:30:16,499 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731544216499"}]},"ts":"9223372036854775807"} 2024-11-14T00:30:16,501 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T00:30:16,501 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4779df6820910ec85365292104f48afc, NAME => 'TestAcidGuarantees,,1731544188573.4779df6820910ec85365292104f48afc.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T00:30:16,502 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T00:30:16,502 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731544216502"}]},"ts":"9223372036854775807"} 2024-11-14T00:30:16,504 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T00:30:16,505 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=162, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T00:30:16,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 79 msec 2024-11-14T00:30:16,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T00:30:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T00:30:16,536 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T00:30:16,545 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241 (was 239) - Thread LEAK? -, OpenFileDescriptor=459 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1198 (was 1150) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2537 (was 3315) 2024-11-14T00:30:16,545 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T00:30:16,546 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T00:30:16,546 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T00:30:16,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:16,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:16,546 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T00:30:16,546 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T00:30:16,546 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=841754823, stopped=false 2024-11-14T00:30:16,546 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=63baa11b75f3,36271,1731544044347 2024-11-14T00:30:16,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T00:30:16,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T00:30:16,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T00:30:16,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T00:30:16,664 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T00:30:16,665 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T00:30:16,665 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T00:30:16,665 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T00:30:16,665 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T00:30:16,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:16,665 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '63baa11b75f3,33199,1731544045387' ***** 2024-11-14T00:30:16,665 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T00:30:16,666 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T00:30:16,666 INFO [RS:0;63baa11b75f3:33199 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T00:30:16,666 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T00:30:16,666 INFO [RS:0;63baa11b75f3:33199 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T00:30:16,666 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(959): stopping server 63baa11b75f3,33199,1731544045387 2024-11-14T00:30:16,666 INFO [RS:0;63baa11b75f3:33199 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T00:30:16,666 INFO [RS:0;63baa11b75f3:33199 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;63baa11b75f3:33199. 2024-11-14T00:30:16,666 DEBUG [RS:0;63baa11b75f3:33199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T00:30:16,666 DEBUG [RS:0;63baa11b75f3:33199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:16,666 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T00:30:16,666 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T00:30:16,666 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T00:30:16,666 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T00:30:16,667 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T00:30:16,667 DEBUG [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T00:30:16,668 DEBUG [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T00:30:16,668 DEBUG [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T00:30:16,668 INFO [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T00:30:16,668 DEBUG [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T00:30:16,668 DEBUG [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T00:30:16,668 DEBUG [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T00:30:16,668 INFO [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-14T00:30:16,687 INFO [regionserver/63baa11b75f3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T00:30:16,691 DEBUG [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/.tmp/info/91602d982e1e496382b3bef59ee83c66 is 95, key is TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2./info:/1731544077487/DeleteFamily/seqid=0 2024-11-14T00:30:16,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742442_1618 (size=6514) 2024-11-14T00:30:16,868 DEBUG [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T00:30:17,068 DEBUG [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T00:30:17,095 INFO [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/.tmp/info/91602d982e1e496382b3bef59ee83c66 2024-11-14T00:30:17,119 DEBUG [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/.tmp/ns/e3cde9b1ecd145ad8f9a76fca58bcf2d is 93, key is TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2./ns:/1731544077389/DeleteFamily/seqid=0 2024-11-14T00:30:17,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742443_1619 (size=5906) 2024-11-14T00:30:17,268 DEBUG [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T00:30:17,469 DEBUG [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T00:30:17,523 INFO [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/.tmp/ns/e3cde9b1ecd145ad8f9a76fca58bcf2d 2024-11-14T00:30:17,533 INFO [regionserver/63baa11b75f3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T00:30:17,533 INFO [regionserver/63baa11b75f3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T00:30:17,546 DEBUG [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/.tmp/rep_barrier/0de0e6d05c774c82973be40de02ce132 is 102, key is TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2./rep_barrier:/1731544077389/DeleteFamily/seqid=0 2024-11-14T00:30:17,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742444_1620 (size=6025) 2024-11-14T00:30:17,669 DEBUG [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T00:30:17,869 DEBUG [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T00:30:17,949 INFO [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/.tmp/rep_barrier/0de0e6d05c774c82973be40de02ce132 2024-11-14T00:30:17,987 DEBUG [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/.tmp/table/3029eadaf61b4c42b0bfff93f9f40cb8 is 96, key is TestAcidGuarantees,,1731544050191.285207bf1146874945959d9159bd86b2./table:/1731544077389/DeleteFamily/seqid=0 2024-11-14T00:30:17,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742445_1621 (size=5936) 2024-11-14T00:30:18,069 DEBUG [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T00:30:18,270 DEBUG [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T00:30:18,398 INFO [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/.tmp/table/3029eadaf61b4c42b0bfff93f9f40cb8 2024-11-14T00:30:18,403 DEBUG [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/.tmp/info/91602d982e1e496382b3bef59ee83c66 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/info/91602d982e1e496382b3bef59ee83c66 2024-11-14T00:30:18,406 INFO [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/info/91602d982e1e496382b3bef59ee83c66, entries=12, sequenceid=90, filesize=6.4 K 2024-11-14T00:30:18,406 DEBUG [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/.tmp/ns/e3cde9b1ecd145ad8f9a76fca58bcf2d as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/ns/e3cde9b1ecd145ad8f9a76fca58bcf2d 2024-11-14T00:30:18,410 INFO [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/ns/e3cde9b1ecd145ad8f9a76fca58bcf2d, entries=8, sequenceid=90, filesize=5.8 K 2024-11-14T00:30:18,411 DEBUG [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/.tmp/rep_barrier/0de0e6d05c774c82973be40de02ce132 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/rep_barrier/0de0e6d05c774c82973be40de02ce132 2024-11-14T00:30:18,414 INFO [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/rep_barrier/0de0e6d05c774c82973be40de02ce132, entries=6, sequenceid=90, filesize=5.9 K 2024-11-14T00:30:18,415 DEBUG [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/.tmp/table/3029eadaf61b4c42b0bfff93f9f40cb8 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/table/3029eadaf61b4c42b0bfff93f9f40cb8 2024-11-14T00:30:18,419 INFO [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/table/3029eadaf61b4c42b0bfff93f9f40cb8, entries=7, sequenceid=90, filesize=5.8 K 2024-11-14T00:30:18,420 INFO [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1752ms, sequenceid=90, compaction requested=false 2024-11-14T00:30:18,431 DEBUG [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-14T00:30:18,432 DEBUG [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T00:30:18,432 INFO [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T00:30:18,432 DEBUG [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731544216667Running coprocessor pre-close hooks at 1731544216668 (+1 ms)Disabling compacts and flushes for region at 1731544216668Disabling writes for close at 1731544216668Obtaining lock to block concurrent updates at 1731544216668Preparing flush snapshotting stores in 1588230740 at 1731544216668Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731544216668Flushing stores of hbase:meta,,1.1588230740 at 1731544216669 (+1 ms)Flushing 1588230740/info: creating writer at 1731544216669Flushing 1588230740/info: appending metadata at 1731544216691 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731544216691Flushing 1588230740/ns: creating writer at 1731544217099 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731544217119 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731544217119Flushing 1588230740/rep_barrier: creating writer at 1731544217526 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731544217545 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731544217545Flushing 1588230740/table: creating writer at 1731544217952 (+407 ms)Flushing 1588230740/table: appending metadata at 1731544217987 (+35 ms)Flushing 1588230740/table: closing flushed file at 1731544217987Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1898eba5: reopening flushed file at 1731544218402 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30bbf192: reopening flushed file at 1731544218406 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70e73886: reopening flushed file at 1731544218410 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71ea5c53: reopening flushed file at 1731544218414 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1752ms, sequenceid=90, compaction requested=false at 1731544218420 (+6 ms)Writing region close event to WAL at 1731544218426 (+6 ms)Running coprocessor post-close hooks at 1731544218431 (+5 ms)Closed at 1731544218432 (+1 ms) 2024-11-14T00:30:18,432 DEBUG [RS_CLOSE_META-regionserver/63baa11b75f3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T00:30:18,471 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(976): stopping server 63baa11b75f3,33199,1731544045387; all regions closed. 2024-11-14T00:30:18,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741834_1010 (size=25344) 2024-11-14T00:30:18,480 DEBUG [RS:0;63baa11b75f3:33199 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/oldWALs 2024-11-14T00:30:18,481 INFO [RS:0;63baa11b75f3:33199 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 63baa11b75f3%2C33199%2C1731544045387.meta:.meta(num 1731544049294) 2024-11-14T00:30:18,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741833_1009 (size=13927495) 2024-11-14T00:30:18,491 DEBUG [RS:0;63baa11b75f3:33199 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/oldWALs 2024-11-14T00:30:18,491 INFO [RS:0;63baa11b75f3:33199 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 63baa11b75f3%2C33199%2C1731544045387:(num 1731544048797) 2024-11-14T00:30:18,491 DEBUG [RS:0;63baa11b75f3:33199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T00:30:18,491 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T00:30:18,491 INFO [RS:0;63baa11b75f3:33199 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T00:30:18,492 INFO [RS:0;63baa11b75f3:33199 {}] hbase.ChoreService(370): Chore service for: regionserver/63baa11b75f3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T00:30:18,492 INFO [RS:0;63baa11b75f3:33199 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T00:30:18,492 INFO [regionserver/63baa11b75f3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T00:30:18,493 INFO [RS:0;63baa11b75f3:33199 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33199 2024-11-14T00:30:18,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T00:30:18,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/63baa11b75f3,33199,1731544045387 2024-11-14T00:30:18,506 INFO [RS:0;63baa11b75f3:33199 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T00:30:18,517 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [63baa11b75f3,33199,1731544045387] 2024-11-14T00:30:18,527 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/63baa11b75f3,33199,1731544045387 already deleted, retry=false 2024-11-14T00:30:18,527 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 63baa11b75f3,33199,1731544045387 expired; onlineServers=0 2024-11-14T00:30:18,527 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '63baa11b75f3,36271,1731544044347' ***** 2024-11-14T00:30:18,527 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T00:30:18,527 INFO [M:0;63baa11b75f3:36271 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T00:30:18,527 INFO [M:0;63baa11b75f3:36271 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T00:30:18,528 DEBUG [M:0;63baa11b75f3:36271 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T00:30:18,528 DEBUG [M:0;63baa11b75f3:36271 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T00:30:18,528 DEBUG [master/63baa11b75f3:0:becomeActiveMaster-HFileCleaner.small.0-1731544048200 {}] cleaner.HFileCleaner(306): Exit Thread[master/63baa11b75f3:0:becomeActiveMaster-HFileCleaner.small.0-1731544048200,5,FailOnTimeoutGroup] 2024-11-14T00:30:18,528 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T00:30:18,528 DEBUG [master/63baa11b75f3:0:becomeActiveMaster-HFileCleaner.large.0-1731544048193 {}] cleaner.HFileCleaner(306): Exit Thread[master/63baa11b75f3:0:becomeActiveMaster-HFileCleaner.large.0-1731544048193,5,FailOnTimeoutGroup] 2024-11-14T00:30:18,528 INFO [M:0;63baa11b75f3:36271 {}] hbase.ChoreService(370): Chore service for: master/63baa11b75f3:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T00:30:18,528 INFO [M:0;63baa11b75f3:36271 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T00:30:18,528 DEBUG [M:0;63baa11b75f3:36271 {}] master.HMaster(1795): Stopping service threads 2024-11-14T00:30:18,528 INFO [M:0;63baa11b75f3:36271 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T00:30:18,528 INFO [M:0;63baa11b75f3:36271 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T00:30:18,528 ERROR [M:0;63baa11b75f3:36271 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (587671560) connection to localhost/127.0.0.1:46041 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:46041,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-14T00:30:18,529 INFO [M:0;63baa11b75f3:36271 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T00:30:18,529 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T00:30:18,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T00:30:18,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T00:30:18,538 DEBUG [M:0;63baa11b75f3:36271 {}] zookeeper.ZKUtil(347): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T00:30:18,538 WARN [M:0;63baa11b75f3:36271 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T00:30:18,538 INFO [M:0;63baa11b75f3:36271 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/.lastflushedseqids 2024-11-14T00:30:18,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742446_1622 (size=72) 2024-11-14T00:30:18,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T00:30:18,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x10136bd4ad10001, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T00:30:18,618 INFO [RS:0;63baa11b75f3:33199 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T00:30:18,618 INFO [RS:0;63baa11b75f3:33199 {}] regionserver.HRegionServer(1031): Exiting; stopping=63baa11b75f3,33199,1731544045387; zookeeper connection closed. 2024-11-14T00:30:18,618 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@44ac23ac {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@44ac23ac 2024-11-14T00:30:18,619 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T00:30:18,961 INFO [M:0;63baa11b75f3:36271 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T00:30:18,961 INFO [M:0;63baa11b75f3:36271 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T00:30:18,961 DEBUG [M:0;63baa11b75f3:36271 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T00:30:18,962 INFO [M:0;63baa11b75f3:36271 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T00:30:18,962 DEBUG [M:0;63baa11b75f3:36271 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T00:30:18,962 DEBUG [M:0;63baa11b75f3:36271 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T00:30:18,962 DEBUG [M:0;63baa11b75f3:36271 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T00:30:18,962 INFO [M:0;63baa11b75f3:36271 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=762.43 KB heapSize=937.57 KB 2024-11-14T00:30:18,981 DEBUG [M:0;63baa11b75f3:36271 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6ef6a25be8e041219bdfbf57026efe8d is 82, key is hbase:meta,,1/info:regioninfo/1731544049463/Put/seqid=0 2024-11-14T00:30:18,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742447_1623 (size=5672) 2024-11-14T00:30:19,384 INFO [M:0;63baa11b75f3:36271 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2172 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6ef6a25be8e041219bdfbf57026efe8d 2024-11-14T00:30:19,414 DEBUG [M:0;63baa11b75f3:36271 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6ad6cff058e04d78ad096cf7fcae5124 is 2276, key is \x00\x00\x00\x00\x00\x00\x00T/proc:d/1731544136868/Put/seqid=0 2024-11-14T00:30:19,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742448_1624 (size=42804) 2024-11-14T00:30:19,444 INFO [M:0;63baa11b75f3:36271 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=761.88 KB at sequenceid=2172 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6ad6cff058e04d78ad096cf7fcae5124 2024-11-14T00:30:19,458 INFO [M:0;63baa11b75f3:36271 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6ad6cff058e04d78ad096cf7fcae5124 2024-11-14T00:30:19,491 DEBUG [M:0;63baa11b75f3:36271 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2306b7cd9bec40bd87f03e19d05e9e58 is 69, key is 63baa11b75f3,33199,1731544045387/rs:state/1731544048214/Put/seqid=0 2024-11-14T00:30:19,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073742449_1625 (size=5156) 2024-11-14T00:30:19,925 INFO [M:0;63baa11b75f3:36271 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2172 (bloomFilter=true), to=hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2306b7cd9bec40bd87f03e19d05e9e58 2024-11-14T00:30:19,940 DEBUG [M:0;63baa11b75f3:36271 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6ef6a25be8e041219bdfbf57026efe8d as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6ef6a25be8e041219bdfbf57026efe8d 2024-11-14T00:30:19,947 INFO [M:0;63baa11b75f3:36271 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6ef6a25be8e041219bdfbf57026efe8d, entries=8, sequenceid=2172, filesize=5.5 K 2024-11-14T00:30:19,948 DEBUG [M:0;63baa11b75f3:36271 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6ad6cff058e04d78ad096cf7fcae5124 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6ad6cff058e04d78ad096cf7fcae5124 2024-11-14T00:30:19,951 INFO [M:0;63baa11b75f3:36271 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6ad6cff058e04d78ad096cf7fcae5124 2024-11-14T00:30:19,952 INFO [M:0;63baa11b75f3:36271 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6ad6cff058e04d78ad096cf7fcae5124, entries=162, sequenceid=2172, filesize=41.8 K 2024-11-14T00:30:19,952 DEBUG [M:0;63baa11b75f3:36271 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2306b7cd9bec40bd87f03e19d05e9e58 as hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2306b7cd9bec40bd87f03e19d05e9e58 2024-11-14T00:30:19,955 INFO [M:0;63baa11b75f3:36271 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46041/user/jenkins/test-data/39c4ee8b-b699-a162-c9f5-d7f302b0193c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2306b7cd9bec40bd87f03e19d05e9e58, entries=1, sequenceid=2172, filesize=5.0 K 2024-11-14T00:30:19,956 INFO [M:0;63baa11b75f3:36271 {}] regionserver.HRegion(3140): Finished flush of dataSize ~762.43 KB/780733, heapSize ~937.27 KB/959768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 994ms, sequenceid=2172, compaction requested=false 2024-11-14T00:30:19,964 INFO [M:0;63baa11b75f3:36271 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T00:30:19,964 DEBUG [M:0;63baa11b75f3:36271 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731544218961Disabling compacts and flushes for region at 1731544218961Disabling writes for close at 1731544218962 (+1 ms)Obtaining lock to block concurrent updates at 1731544218962Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731544218962Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=780733, getHeapSize=960008, getOffHeapSize=0, getCellsCount=2359 at 1731544218962Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731544218963 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731544218963Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731544218980 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731544218980Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731544219388 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731544219413 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731544219413Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731544219458 (+45 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731544219490 (+32 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731544219490Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11bb65b4: reopening flushed file at 1731544219933 (+443 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12e47786: reopening flushed file at 1731544219948 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2faa181: reopening flushed file at 1731544219952 (+4 ms)Finished flush of dataSize ~762.43 KB/780733, heapSize ~937.27 KB/959768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 994ms, sequenceid=2172, compaction requested=false at 1731544219956 (+4 ms)Writing region close event to WAL at 1731544219964 (+8 ms)Closed at 1731544219964 2024-11-14T00:30:19,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43405 is added to blk_1073741830_1006 (size=923080) 2024-11-14T00:30:20,001 INFO [M:0;63baa11b75f3:36271 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T00:30:20,001 INFO [M:0;63baa11b75f3:36271 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36271 2024-11-14T00:30:20,001 INFO [M:0;63baa11b75f3:36271 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T00:30:20,001 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T00:30:20,265 INFO [M:0;63baa11b75f3:36271 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T00:30:20,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T00:30:20,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36271-0x10136bd4ad10000, quorum=127.0.0.1:56442, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T00:30:20,274 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@46a94ab{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T00:30:20,277 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@753587b3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T00:30:20,277 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T00:30:20,277 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b5418b0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T00:30:20,277 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@24457f8e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/hadoop.log.dir/,STOPPED} 2024-11-14T00:30:20,282 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T00:30:20,282 WARN [BP-301748371-172.17.0.3-1731544037581 heartbeating to localhost/127.0.0.1:46041 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T00:30:20,282 WARN [BP-301748371-172.17.0.3-1731544037581 heartbeating to localhost/127.0.0.1:46041 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-301748371-172.17.0.3-1731544037581 (Datanode Uuid 16a71a9d-0b02-43b0-8a7a-4cb8cb1ab9d4) service to localhost/127.0.0.1:46041 2024-11-14T00:30:20,282 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T00:30:20,286 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/cluster_41f6304e-0c68-787b-c426-53235df1090e/data/data1/current/BP-301748371-172.17.0.3-1731544037581 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T00:30:20,286 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/cluster_41f6304e-0c68-787b-c426-53235df1090e/data/data2/current/BP-301748371-172.17.0.3-1731544037581 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T00:30:20,287 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T00:30:20,318 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7f405bdd{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T00:30:20,319 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5b829fcb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T00:30:20,319 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T00:30:20,320 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31a6a643{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T00:30:20,320 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@610d6b27{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8d758787-0b8f-8894-d35e-c2154fd930d5/hadoop.log.dir/,STOPPED} 2024-11-14T00:30:20,343 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T00:30:20,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down